From 7fdc968227e0193b93fe964629413c612560a613 Mon Sep 17 00:00:00 2001 From: battlmonstr Date: Wed, 4 Dec 2024 14:22:09 +0100 Subject: [PATCH] db: fix datastore namespaces This is a preparation before adding DHII into datastore/mdbx. Everything inside datastore/mdbx should have the same namespace independent of silkworm::db. The silkworm::db contains DAL while datastore/mdbx contains the datastore implementation. * move top level datastore items to silkworm::datastore * move items in datastore/mdbx from silkworm::db to silkworm::sw_mdbx * move items in datastore/etl to silkworm::etl Note 1: a name "sw_mdbx" is chosen to avoid a conflict with ::mdbx own namespace. Note 2: code that uses access_layer.hpp or stage.hpp will use aliases of ROTxn/RWTxn inside silkworm::db namespace. This is done to avoid extra churn in code that we don't actively support. --- cmd/capi/execute.cpp | 12 +- cmd/common/db_max_readers_option.cpp | 2 +- cmd/dev/backend_kv_server.cpp | 14 +- cmd/dev/check_blockhashes.cpp | 10 +- cmd/dev/check_changes.cpp | 4 +- cmd/dev/check_hashstate.cpp | 1 + cmd/dev/check_log_indices.cpp | 1 + cmd/dev/check_pow.cpp | 12 +- cmd/dev/check_senders.cpp | 1 + cmd/dev/check_tx_lookup.cpp | 22 +-- cmd/dev/db_toolbox.cpp | 3 +- cmd/dev/scan_txs.cpp | 6 +- cmd/dev/snapshots.cpp | 6 +- cmd/dev/staged_pipeline.cpp | 139 +++++++++--------- cmd/silkworm.cpp | 2 +- silkworm/capi/fork_validator.cpp | 10 +- silkworm/capi/rpcdaemon.cpp | 2 +- silkworm/capi/silkworm.cpp | 14 +- silkworm/capi/silkworm_test.cpp | 1 + silkworm/db/CMakeLists.txt | 2 +- silkworm/db/access_layer.cpp | 2 + silkworm/db/access_layer.hpp | 5 +- silkworm/db/access_layer_test.cpp | 3 + silkworm/db/bitmap_test.cpp | 8 +- .../blocks/bodies/body_segment_collation.cpp | 3 + .../blocks/bodies/body_segment_collation.hpp | 6 +- .../headers/header_segment_collation.cpp | 3 + .../headers/header_segment_collation.hpp | 6 +- silkworm/db/blocks/transactions/txn_index.hpp | 2 +- .../transactions/txn_segment_collation.cpp | 3 + .../transactions/txn_segment_collation.hpp | 6 +- .../transactions/txn_to_block_index.hpp | 2 +- silkworm/db/buffer.cpp | 2 + silkworm/db/buffer_test.cpp | 1 + silkworm/db/chain_data_init.cpp | 4 +- silkworm/db/chain_data_init.hpp | 2 +- silkworm/db/chain_head.cpp | 4 +- silkworm/db/chain_head.hpp | 4 +- silkworm/db/data_store.hpp | 10 +- silkworm/db/datastore/CMakeLists.txt | 2 +- silkworm/db/datastore/data_migration.cpp | 4 +- silkworm/db/datastore/data_migration.hpp | 4 +- .../db/datastore/data_migration_command.hpp | 4 +- silkworm/db/datastore/data_store.hpp | 4 +- silkworm/db/datastore/etl/CMakeLists.txt | 2 +- silkworm/db/datastore/etl/buffer.hpp | 4 +- silkworm/db/datastore/etl/collector.cpp | 4 +- silkworm/db/datastore/etl/collector.hpp | 4 +- .../db/datastore/etl/collector_settings.hpp | 4 +- silkworm/db/datastore/etl/file_provider.cpp | 4 +- silkworm/db/datastore/etl/file_provider.hpp | 4 +- .../db/datastore/etl/in_memory_collector.hpp | 4 +- silkworm/db/datastore/etl/util.hpp | 4 +- silkworm/db/datastore/etl/util_test.cpp | 4 +- silkworm/db/datastore/mdbx/CMakeLists.txt | 6 +- silkworm/db/datastore/mdbx/bitmap.cpp | 36 ++--- silkworm/db/datastore/mdbx/bitmap.hpp | 24 +-- .../db/datastore/mdbx/etl_mdbx_collector.hpp | 12 +- silkworm/db/datastore/mdbx/mdbx.cpp | 10 +- silkworm/db/datastore/mdbx/mdbx.hpp | 8 +- silkworm/db/datastore/mdbx/mdbx_test.cpp | 4 +- .../db/datastore/mdbx/memory_mutation.cpp | 22 +-- .../db/datastore/mdbx/memory_mutation.hpp | 10 +- .../datastore/mdbx/memory_mutation_cursor.cpp | 4 +- .../datastore/mdbx/memory_mutation_cursor.hpp | 4 +- .../datastore/mdbx/memory_mutation_test.cpp | 10 +- silkworm/db/datastore/segment_collation.hpp | 8 +- silkworm/db/datastore/snapshot_merger.cpp | 4 +- silkworm/db/datastore/snapshot_merger.hpp | 4 +- .../db/datastore/snapshots/index_builder.hpp | 2 +- .../snapshots/rec_split/rec_split_seq.hpp | 16 +- .../rec_split/rec_split_seq_test.cpp | 2 +- .../snapshots/segment/seg/CMakeLists.txt | 2 +- .../seg/compressor/pattern_aggregator.cpp | 18 +-- silkworm/db/datastore/stage_scheduler.hpp | 6 +- silkworm/db/db_utils.cpp | 8 +- silkworm/db/db_utils.hpp | 6 +- silkworm/db/db_utils_test.cpp | 2 + silkworm/db/etl_collector_test.cpp | 13 +- silkworm/db/etl_in_memory_collector_test.cpp | 10 +- silkworm/db/freezer.cpp | 1 + silkworm/db/freezer.hpp | 15 +- silkworm/db/genesis.cpp | 2 + silkworm/db/genesis.hpp | 1 + silkworm/db/genesis_test.cpp | 2 +- silkworm/db/kv/api/local_cursor.cpp | 3 +- silkworm/db/kv/api/local_cursor.hpp | 2 +- silkworm/db/kv/api/local_cursor_test.cpp | 3 +- silkworm/db/kv/api/local_transaction.hpp | 2 +- silkworm/db/kv/grpc/server/kv_calls.hpp | 2 + silkworm/db/kv/grpc/server/kv_calls_test.cpp | 18 ++- silkworm/db/kv/grpc/server/kv_server.hpp | 4 +- silkworm/db/kv/grpc/server/kv_server_test.cpp | 28 ++-- silkworm/db/mdbx_test.cpp | 6 +- silkworm/db/memory_mutation_cursor_test.cpp | 6 +- silkworm/db/prune_mode.cpp | 7 +- silkworm/db/snapshot_sync.cpp | 8 +- silkworm/db/snapshot_sync.hpp | 4 +- silkworm/db/snapshot_sync_test.cpp | 2 +- silkworm/db/stage.cpp | 10 +- silkworm/db/stage.hpp | 6 + silkworm/db/stages.cpp | 8 +- silkworm/db/stages.hpp | 3 + silkworm/db/tables.cpp | 4 +- silkworm/db/tables.hpp | 4 +- silkworm/db/test_util/mock_ro_cursor.hpp | 11 +- silkworm/db/test_util/mock_txn.hpp | 8 +- silkworm/db/test_util/temp_chain_data.cpp | 8 +- silkworm/db/test_util/temp_chain_data.hpp | 16 +- .../db/test_util/test_database_context.cpp | 4 +- .../db/test_util/test_database_context.hpp | 12 +- silkworm/db/util.cpp | 7 +- silkworm/db/util.hpp | 8 +- silkworm/execution/local_state.hpp | 2 +- silkworm/node/CMakeLists.txt | 2 +- silkworm/node/backend/ethereum_backend.cpp | 4 +- silkworm/node/backend/ethereum_backend.hpp | 8 +- .../node/backend/ethereum_backend_test.cpp | 6 +- silkworm/node/common/node_settings.hpp | 4 +- .../execution/header_chain_plus_exec_test.cpp | 2 +- silkworm/node/node.cpp | 4 +- .../grpc/server/backend_server_test.cpp | 33 +++-- silkworm/node/stagedsync/execution_engine.cpp | 4 +- silkworm/node/stagedsync/execution_engine.hpp | 4 +- .../node/stagedsync/execution_engine_test.cpp | 2 +- .../node/stagedsync/execution_pipeline.cpp | 4 +- .../node/stagedsync/execution_pipeline.hpp | 2 +- .../node/stagedsync/forks/extending_fork.cpp | 2 +- silkworm/node/stagedsync/forks/fork.cpp | 8 +- silkworm/node/stagedsync/forks/fork.hpp | 8 +- silkworm/node/stagedsync/forks/fork_test.cpp | 4 +- silkworm/node/stagedsync/forks/main_chain.cpp | 10 +- silkworm/node/stagedsync/forks/main_chain.hpp | 8 +- .../node/stagedsync/forks/main_chain_test.cpp | 2 +- .../node/stagedsync/stages/CMakeLists.txt | 2 +- .../stagedsync/stages/stage_blockhashes.cpp | 16 +- .../stagedsync/stages/stage_blockhashes.hpp | 8 +- .../stagedsync/stages/stage_bodies_test.cpp | 6 +- .../stages/stage_call_trace_index.cpp | 3 +- .../stages/stage_call_trace_index.hpp | 10 +- .../stagedsync/stages/stage_execution.cpp | 1 + .../stagedsync/stages/stage_execution.hpp | 8 +- .../stagedsync/stages/stage_hashstate.cpp | 20 +-- .../stagedsync/stages/stage_hashstate.hpp | 8 +- .../stagedsync/stages/stage_headers_test.cpp | 6 +- .../stagedsync/stages/stage_history_index.cpp | 7 +- .../stagedsync/stages/stage_history_index.hpp | 8 +- .../stages/stage_history_index_test.cpp | 1 + .../stagedsync/stages/stage_interhashes.cpp | 6 +- .../stagedsync/stages/stage_interhashes.hpp | 10 +- .../stages/stage_interhashes/_test.cpp | 3 +- .../stages/stage_interhashes/trie_cursor.cpp | 12 +- .../stages/stage_interhashes/trie_cursor.hpp | 10 +- .../stages/stage_interhashes/trie_loader.cpp | 1 + .../stages/stage_interhashes/trie_loader.hpp | 14 +- .../stagedsync/stages/stage_log_index.cpp | 6 +- .../stagedsync/stages/stage_log_index.hpp | 10 +- .../node/stagedsync/stages/stage_senders.cpp | 4 +- .../node/stagedsync/stages/stage_senders.hpp | 6 +- .../node/stagedsync/stages/stage_triggers.hpp | 2 +- .../stagedsync/stages/stage_tx_lookup.cpp | 2 +- .../stagedsync/stages/stage_tx_lookup.hpp | 6 +- .../stages/stage_tx_lookup_test.cpp | 1 + silkworm/node/stagedsync/stages_test.cpp | 3 +- .../node/test_util/mock_execution_engine.hpp | 2 +- silkworm/rpc/commands/ots_api.cpp | 3 + silkworm/rpc/daemon.cpp | 6 +- .../sync/internals/body_sequence_test.cpp | 2 +- .../sync/internals/header_retrieval_test.cpp | 2 +- .../messages/inbound_get_block_bodies.cpp | 2 +- .../messages/inbound_get_block_headers.cpp | 2 +- 171 files changed, 660 insertions(+), 554 deletions(-) diff --git a/cmd/capi/execute.cpp b/cmd/capi/execute.cpp index 0dbb38f237..555de3f3c7 100644 --- a/cmd/capi/execute.cpp +++ b/cmd/capi/execute.cpp @@ -223,8 +223,8 @@ std::vector collect_all_snapshots(const SnapshotRepositor return snapshot_sequence; } -int execute_with_internal_txn(SilkwormHandle handle, ExecuteBlocksSettings settings, db::RWAccess chaindata) { - db::ROTxnManaged ro_txn = chaindata.start_ro_tx(); +int execute_with_internal_txn(SilkwormHandle handle, ExecuteBlocksSettings settings, sw_mdbx::RWAccess chaindata) { + sw_mdbx::ROTxnManaged ro_txn = chaindata.start_ro_tx(); const auto chain_config{db::read_chain_config(ro_txn)}; ensure(chain_config.has_value(), "no chain configuration in database"); const auto chain_id{chain_config->chain_id}; @@ -252,7 +252,7 @@ int execute_with_internal_txn(SilkwormHandle handle, ExecuteBlocksSettings setti return status_code; } -int execute_with_external_txn(SilkwormHandle handle, ExecuteBlocksSettings settings, db::RWTxnManaged rw_txn) { +int execute_with_external_txn(SilkwormHandle handle, ExecuteBlocksSettings settings, sw_mdbx::RWTxnManaged rw_txn) { const auto chain_config{db::read_chain_config(rw_txn)}; ensure(chain_config.has_value(), "no chain configuration in database"); const auto chain_id{chain_config->chain_id}; @@ -283,7 +283,7 @@ int execute_with_external_txn(SilkwormHandle handle, ExecuteBlocksSettings setti int execute_blocks(SilkwormHandle handle, ExecuteBlocksSettings settings, const DataDirectory& data_dir) { // Open chain database - silkworm::db::EnvConfig config{ + silkworm::sw_mdbx::EnvConfig config{ .path = data_dir.chaindata().path().string(), .readonly = false, .exclusive = true}; @@ -376,11 +376,11 @@ int start_rpcdaemon(SilkwormHandle handle, const rpc::DaemonSettings& /*settings }); // Open chain database - silkworm::db::EnvConfig config{ + silkworm::sw_mdbx::EnvConfig config{ .path = data_dir.chaindata().path().string(), .readonly = false, .exclusive = true}; - ::mdbx::env_managed env{silkworm::db::open_env(config)}; + ::mdbx::env_managed env{silkworm::sw_mdbx::open_env(config)}; SilkwormRpcSettings settings{}; const int status_code{silkworm_start_rpcdaemon(handle, &*env, &settings)}; diff --git a/cmd/common/db_max_readers_option.cpp b/cmd/common/db_max_readers_option.cpp index 907b236461..4310d17f2d 100644 --- a/cmd/common/db_max_readers_option.cpp +++ b/cmd/common/db_max_readers_option.cpp @@ -22,7 +22,7 @@ namespace silkworm::cmd::common { void add_option_db_max_readers(CLI::App& cli, uint32_t& max_readers) { cli.add_option("--mdbx.max.readers", max_readers, "The maximum number of MDBX readers") - ->default_val(silkworm::db::EnvConfig{}.max_readers) + ->default_val(silkworm::sw_mdbx::EnvConfig{}.max_readers) ->check(CLI::Range(1, 32767)); } diff --git a/cmd/dev/backend_kv_server.cpp b/cmd/dev/backend_kv_server.cpp index 7d443121f3..978a0b15d4 100644 --- a/cmd/dev/backend_kv_server.cpp +++ b/cmd/dev/backend_kv_server.cpp @@ -102,16 +102,16 @@ void parse_command_line(int argc, char* argv[], CLI::App& app, StandaloneBackEnd } node_settings.data_directory = std::make_unique(data_dir, /*create=*/false); - node_settings.chaindata_env_config = db::EnvConfig{node_settings.data_directory->chaindata().path().string(), - /*create=*/false, - /*readonly=*/true}; + node_settings.chaindata_env_config = sw_mdbx::EnvConfig{node_settings.data_directory->chaindata().path().string(), + /*create=*/false, + /*readonly=*/true}; node_settings.chaindata_env_config.max_readers = max_readers; } std::shared_ptr make_sentry_client( const NodeSettings& node_settings, rpc::ClientContextPool& context_pool, - db::ROAccess db_access) { + sw_mdbx::ROAccess db_access) { std::shared_ptr sentry_client; auto chain_head_provider = [db_access = std::move(db_access)]() { @@ -178,12 +178,12 @@ int main(int argc, char* argv[]) { << " address: " << server_settings.address_uri << " contexts: " << server_settings.context_pool_settings.num_contexts; - auto chaindata_env = db::open_env(node_settings.chaindata_env_config); + auto chaindata_env = sw_mdbx::open_env(node_settings.chaindata_env_config); SILK_INFO << "BackEndKvServer MDBX max readers: " << chaindata_env.max_readers(); // Read chain config from database (this allows for custom config) - db::ROAccess chaindata{chaindata_env}; - db::ROTxnManaged ro_txn = chaindata.start_ro_tx(); + sw_mdbx::ROAccess chaindata{chaindata_env}; + sw_mdbx::ROTxnManaged ro_txn = chaindata.start_ro_tx(); node_settings.chain_config = db::read_chain_config(ro_txn); if (!node_settings.chain_config.has_value()) { throw std::runtime_error("invalid chain config in database"); diff --git a/cmd/dev/check_blockhashes.cpp b/cmd/dev/check_blockhashes.cpp index 603823cb17..4a66870c90 100644 --- a/cmd/dev/check_blockhashes.cpp +++ b/cmd/dev/check_blockhashes.cpp @@ -38,12 +38,12 @@ int main(int argc, char* argv[]) { try { auto data_dir{DataDirectory::from_chaindata(chaindata)}; data_dir.deploy(); - db::EnvConfig db_config{data_dir.chaindata().path().string()}; - auto env{db::open_env(db_config)}; + sw_mdbx::EnvConfig db_config{data_dir.chaindata().path().string()}; + auto env{sw_mdbx::open_env(db_config)}; auto txn{env.start_read()}; - auto canonical_hashes_table{db::open_cursor(txn, db::table::kCanonicalHashes)}; - auto blockhashes_table{db::open_cursor(txn, db::table::kHeaderNumbers)}; + auto canonical_hashes_table = sw_mdbx::open_cursor(txn, db::table::kCanonicalHashes); + auto blockhashes_table = sw_mdbx::open_cursor(txn, db::table::kHeaderNumbers); uint32_t scanned_headers{0}; SILK_INFO << "Checking Block Hashes..."; @@ -54,7 +54,7 @@ int main(int argc, char* argv[]) { // Check if each hash has the correct number according to the header table while (canonical_hashes_data) { - ByteView hash_data_view{db::from_slice(canonical_hashes_data.value)}; // Canonical Hash + ByteView hash_data_view{sw_mdbx::from_slice(canonical_hashes_data.value)}; // Canonical Hash auto block_hashes_data{blockhashes_table.find(canonical_hashes_data.value, /*throw_notfound*/ false)}; if (!block_hashes_data) { uint64_t hash_block_num{ diff --git a/cmd/dev/check_changes.cpp b/cmd/dev/check_changes.cpp index a336b57c22..a82911500a 100644 --- a/cmd/dev/check_changes.cpp +++ b/cmd/dev/check_changes.cpp @@ -102,14 +102,14 @@ int main(int argc, char* argv[]) { auto data_dir{DataDirectory::from_chaindata(chaindata)}; data_dir.deploy(); - db::EnvConfig db_config{data_dir.chaindata().path().string()}; + sw_mdbx::EnvConfig db_config{data_dir.chaindata().path().string()}; db::DataStore data_store{ db_config, data_dir.snapshots().path(), }; - db::RWTxnManaged txn = data_store.chaindata_rw().start_rw_tx(); + auto txn = data_store.chaindata_rw().start_rw_tx(); auto chain_config{db::read_chain_config(txn)}; if (!chain_config) { throw std::runtime_error("Unable to retrieve chain config"); diff --git a/cmd/dev/check_hashstate.cpp b/cmd/dev/check_hashstate.cpp index 8cc8828665..5661302428 100644 --- a/cmd/dev/check_hashstate.cpp +++ b/cmd/dev/check_hashstate.cpp @@ -25,6 +25,7 @@ using namespace silkworm; using namespace silkworm::db; +using namespace silkworm::sw_mdbx; enum Operation { kHashAccount, diff --git a/cmd/dev/check_log_indices.cpp b/cmd/dev/check_log_indices.cpp index 1623403c52..50786265ab 100644 --- a/cmd/dev/check_log_indices.cpp +++ b/cmd/dev/check_log_indices.cpp @@ -44,6 +44,7 @@ using Roaring = roaring::Roaring; using namespace silkworm; using namespace silkworm::db; +using namespace silkworm::sw_mdbx; using namespace silkworm::cmd::common; enum class TargetIndex { diff --git a/cmd/dev/check_pow.cpp b/cmd/dev/check_pow.cpp index a434b2aefe..c137621659 100644 --- a/cmd/dev/check_pow.cpp +++ b/cmd/dev/check_pow.cpp @@ -79,9 +79,9 @@ int main(int argc, char* argv[]) { options.datadir = data_dir.chaindata().path().string(); // Set database parameters - db::EnvConfig db_config{options.datadir}; - auto env{db::open_env(db_config)}; - db::ROTxnManaged txn{env}; + sw_mdbx::EnvConfig db_config{options.datadir}; + auto env{sw_mdbx::open_env(db_config)}; + sw_mdbx::ROTxnManaged txn{env}; auto config{db::read_chain_config(txn)}; if (!config.has_value()) { @@ -99,7 +99,7 @@ int main(int argc, char* argv[]) { SILK_INFO << "Initializing Light Cache for DAG epoch " << epoch_num; auto epoch_context{ethash::create_epoch_context(static_cast(epoch_num))}; - auto canonical_hashes{db::open_cursor(txn, db::table::kCanonicalHashes)}; + auto canonical_hashes = sw_mdbx::open_cursor(txn, db::table::kCanonicalHashes); // Loop blocks StopWatch sw; @@ -113,12 +113,12 @@ int main(int argc, char* argv[]) { } auto block_key{db::block_key(block_num)}; - auto data{canonical_hashes.find(db::to_slice(block_key), /*throw_notfound*/ false)}; + auto data{canonical_hashes.find(sw_mdbx::to_slice(block_key), /*throw_notfound*/ false)}; if (!data) { throw std::runtime_error("Can't retrieve canonical hash for block " + std::to_string(block_num)); } - auto header_key{to_bytes32(db::from_slice(data.value))}; + auto header_key{to_bytes32(sw_mdbx::from_slice(data.value))}; auto header{db::read_header(txn, block_num, header_key.bytes)}; if (!header.has_value()) { throw std::runtime_error("Can't retrieve header for block " + std::to_string(block_num)); diff --git a/cmd/dev/check_senders.cpp b/cmd/dev/check_senders.cpp index d30812c52e..ed7307976b 100644 --- a/cmd/dev/check_senders.cpp +++ b/cmd/dev/check_senders.cpp @@ -35,6 +35,7 @@ using namespace silkworm; using namespace silkworm::db; +using namespace silkworm::sw_mdbx; using namespace silkworm::cmd::common; int main(int argc, char* argv[]) { diff --git a/cmd/dev/check_tx_lookup.cpp b/cmd/dev/check_tx_lookup.cpp index d5e5cd2c17..98bd676086 100644 --- a/cmd/dev/check_tx_lookup.cpp +++ b/cmd/dev/check_tx_lookup.cpp @@ -46,15 +46,15 @@ int main(int argc, char* argv[]) { auto data_dir{DataDirectory::from_chaindata(chaindata)}; data_dir.deploy(); - db::EnvConfig db_config{data_dir.chaindata().path().string()}; - db::etl::Collector collector(data_dir.temp().path().string().c_str(), /* flush size */ 512 * kMebi); + sw_mdbx::EnvConfig db_config{data_dir.chaindata().path().string()}; + etl::Collector collector(data_dir.temp().path().string().c_str(), /* flush size */ 512 * kMebi); - auto env{db::open_env(db_config)}; + auto env{sw_mdbx::open_env(db_config)}; auto txn{env.start_read()}; - auto bodies_table{db::open_cursor(txn, db::table::kBlockBodies)}; - auto tx_lookup_table{db::open_cursor(txn, db::table::kTxLookup)}; - auto transactions_table{db::open_cursor(txn, db::table::kBlockTransactions)}; + auto bodies_table = sw_mdbx::open_cursor(txn, db::table::kBlockBodies); + auto tx_lookup_table = sw_mdbx::open_cursor(txn, db::table::kTxLookup); + auto transactions_table = sw_mdbx::open_cursor(txn, db::table::kBlockTransactions); uint64_t expected_block_num{0}; @@ -64,7 +64,7 @@ int main(int argc, char* argv[]) { auto bodies_data{bodies_table.to_first(false)}; while (bodies_data) { auto block_num(endian::load_big_u64(static_cast(bodies_data.key.data()))); - auto body_rlp{db::from_slice(bodies_data.value)}; + auto body_rlp{sw_mdbx::from_slice(bodies_data.value)}; auto body{unwrap_or_throw(decode_stored_block_body(body_rlp))}; if (body.txn_count > 0) { @@ -72,7 +72,7 @@ int main(int argc, char* argv[]) { endian::store_big_u64(transaction_key.data(), body.base_txn_id); uint64_t i{0}; - auto transaction_data{transactions_table.find(db::to_slice(transaction_key), false)}; + auto transaction_data{transactions_table.find(sw_mdbx::to_slice(transaction_key), false)}; for (; i < body.txn_count && transaction_data.done; ++i, transaction_data = transactions_table.to_next(false)) { if (!transaction_data) { @@ -81,10 +81,10 @@ int main(int argc, char* argv[]) { continue; } - ByteView transaction_rlp{db::from_slice(transaction_data.value)}; + ByteView transaction_rlp{sw_mdbx::from_slice(transaction_data.value)}; auto transaction_hash{keccak256(transaction_rlp)}; ByteView transaction_view{transaction_hash.bytes}; - auto lookup_data{tx_lookup_table.find(db::to_slice(transaction_view), false)}; + auto lookup_data{tx_lookup_table.find(sw_mdbx::to_slice(transaction_view), false)}; if (!lookup_data) { SILK_ERROR << "Block " << block_num << " transaction " << i << " with hash " << to_hex(transaction_view) << " not found in " << db::table::kTxLookup.name @@ -93,7 +93,7 @@ int main(int argc, char* argv[]) { } // Erigon stores block_num as compact (no leading zeroes) - auto lookup_block_value{db::from_slice(lookup_data.value)}; + auto lookup_block_value{sw_mdbx::from_slice(lookup_data.value)}; uint64_t actual_block_num{0}; if (!endian::from_big_compact(lookup_block_value, actual_block_num)) { SILK_ERROR << "Failed to read expected block number from: " << to_hex(lookup_block_value); diff --git a/cmd/dev/db_toolbox.cpp b/cmd/dev/db_toolbox.cpp index 1c7bac182c..c9f896f090 100644 --- a/cmd/dev/db_toolbox.cpp +++ b/cmd/dev/db_toolbox.cpp @@ -59,6 +59,7 @@ namespace fs = std::filesystem; using namespace silkworm; using namespace silkworm::db; +using namespace silkworm::sw_mdbx; class Progress { public: @@ -1426,7 +1427,7 @@ void do_extract_headers(EnvConfig& config, const std::string& file_name, uint32_ void do_freeze(EnvConfig& config, const DataDirectory& data_dir, bool keep_blocks) { using namespace concurrency::awaitable_wait_for_one; - class StageSchedulerAdapter : public stagedsync::StageScheduler, public ActiveComponent { + class StageSchedulerAdapter : public datastore::StageScheduler, public ActiveComponent { public: explicit StageSchedulerAdapter(RWAccess db_access) : db_access_(std::move(db_access)) {} diff --git a/cmd/dev/scan_txs.cpp b/cmd/dev/scan_txs.cpp index 2608adb699..75e4ebb673 100644 --- a/cmd/dev/scan_txs.cpp +++ b/cmd/dev/scan_txs.cpp @@ -61,9 +61,9 @@ int main(int argc, char* argv[]) { try { auto data_dir{DataDirectory::from_chaindata(chaindata)}; data_dir.deploy(); - db::EnvConfig db_config{data_dir.chaindata().path().string()}; - auto env{db::open_env(db_config)}; - db::RWTxnManaged txn{env}; + sw_mdbx::EnvConfig db_config{data_dir.chaindata().path().string()}; + auto env{sw_mdbx::open_env(db_config)}; + sw_mdbx::RWTxnManaged txn{env}; auto chain_config{db::read_chain_config(txn)}; if (!chain_config) { throw std::runtime_error("Unable to retrieve chain config"); diff --git a/cmd/dev/snapshots.cpp b/cmd/dev/snapshots.cpp index bfa65c6553..f92551ef33 100644 --- a/cmd/dev/snapshots.cpp +++ b/cmd/dev/snapshots.cpp @@ -1001,13 +1001,13 @@ void lookup_transaction(const SnapshotSubcommandSettings& settings) { void merge(const SnapshotSettings& settings) { auto repository = make_repository(settings); TemporaryDirectory tmp_dir; - db::SnapshotMerger merger{repository, tmp_dir.path()}; + datastore::SnapshotMerger merger{repository, tmp_dir.path()}; test_util::TaskRunner runner; runner.run(merger.exec()); } void sync(const SnapshotSettings& settings) { - class NoopStageSchedulerAdapter : public stagedsync::StageScheduler { + class NoopStageSchedulerAdapter : public datastore::StageScheduler { public: explicit NoopStageSchedulerAdapter() = default; ~NoopStageSchedulerAdapter() override = default; @@ -1019,7 +1019,7 @@ void sync(const SnapshotSettings& settings) { std::chrono::time_point start{std::chrono::steady_clock::now()}; TemporaryDirectory tmp_dir; - db::EnvConfig chaindata_env_config{tmp_dir.path()}; + sw_mdbx::EnvConfig chaindata_env_config{tmp_dir.path()}; db::DataStore data_store{ chaindata_env_config, diff --git a/cmd/dev/staged_pipeline.cpp b/cmd/dev/staged_pipeline.cpp index a320d90578..0042ee07a8 100644 --- a/cmd/dev/staged_pipeline.cpp +++ b/cmd/dev/staged_pipeline.cpp @@ -58,6 +58,7 @@ namespace fs = std::filesystem; using namespace silkworm; +using namespace silkworm::sw_mdbx; class Progress { public: @@ -119,11 +120,11 @@ class Progress { uint32_t printed_bar_len_{0}; }; -void cursor_for_each(::mdbx::cursor& cursor, db::WalkFuncRef walker) { +void cursor_for_each(::mdbx::cursor& cursor, sw_mdbx::WalkFuncRef walker) { const bool throw_notfound{false}; auto data = cursor.eof() ? cursor.to_first(throw_notfound) : cursor.current(throw_notfound); while (data) { - walker(db::from_slice(data.key), db::from_slice(data.value)); + walker(sw_mdbx::from_slice(data.key), sw_mdbx::from_slice(data.value)); data = cursor.move(mdbx::cursor::move_operation::next, throw_notfound); } } @@ -172,18 +173,18 @@ struct StageOrderCompare { } }; -void list_stages(db::EnvConfig& config) { +void list_stages(sw_mdbx::EnvConfig& config) { static constexpr char kTableHeaderFormat[] = " %-26s %10s "; static constexpr char kTableRowFormat[] = " %-26s %10u %-8s"; - auto env = silkworm::db::open_env(config); + auto env = silkworm::sw_mdbx::open_env(config); auto txn = env.start_read(); - if (!db::has_map(txn, db::table::kSyncStageProgress.name)) { + if (!has_map(txn, db::table::kSyncStageProgress.name)) { throw std::runtime_error("Either not a Silkworm db or table " + std::string{db::table::kSyncStageProgress.name} + " not found"); } - auto stage_cursor = db::open_cursor(txn, db::table::kSyncStageProgress); + auto stage_cursor = open_cursor(txn, db::table::kSyncStageProgress); if (txn.get_map_stat(stage_cursor.map()).ms_entries) { std::map stage_height_by_name; auto result = stage_cursor.to_first(/*throw_notfound =*/false); @@ -219,19 +220,19 @@ void list_stages(db::EnvConfig& config) { env.close(config.shared); } -void set_stage_progress(db::EnvConfig& config, const std::string& stage_name, uint32_t new_height, bool dry) { +void set_stage_progress(sw_mdbx::EnvConfig& config, const std::string& stage_name, uint32_t new_height, bool dry) { config.readonly = false; if (!config.exclusive) { throw std::runtime_error("Function requires exclusive access to database"); } - auto env{silkworm::db::open_env(config)}; - db::RWTxnManaged txn{env}; + auto env{silkworm::sw_mdbx::open_env(config)}; + RWTxnManaged txn{env}; if (!db::stages::is_known_stage(stage_name.c_str())) { throw std::runtime_error("Stage name " + stage_name + " is not known"); } - if (!db::has_map(txn, silkworm::db::table::kSyncStageProgress.name)) { + if (!has_map(txn, silkworm::db::table::kSyncStageProgress.name)) { throw std::runtime_error("Either non Silkworm db or table " + std::string(silkworm::db::table::kSyncStageProgress.name) + " not found"); } @@ -267,7 +268,7 @@ static stagedsync::StageContainerFactory make_stages_factory( }); } -void debug_unwind(db::EnvConfig& config, BlockNum height, uint32_t step, const bool dry, const bool force, +void debug_unwind(sw_mdbx::EnvConfig& config, BlockNum height, uint32_t step, const bool dry, const bool force, const std::string& start_at_stage, const std::string& stop_before_stage) { ensure(config.exclusive, "Function requires exclusive access to database"); ensure(height > 0, "Function requires non-zero height block"); @@ -281,9 +282,9 @@ void debug_unwind(db::EnvConfig& config, BlockNum height, uint32_t step, const b Environment::set_start_at_stage(start_at_stage); Environment::set_stop_before_stage(stop_before_stage); - auto env = silkworm::db::open_env(config); + auto env = silkworm::sw_mdbx::open_env(config); - db::ROTxnManaged ro_txn{env}; + sw_mdbx::ROTxnManaged ro_txn{env}; const auto chain_config = db::read_chain_config(ro_txn); ensure(chain_config.has_value(), "Uninitialized Silkworm db or unknown/custom chain"); ro_txn.abort(); @@ -291,7 +292,7 @@ void debug_unwind(db::EnvConfig& config, BlockNum height, uint32_t step, const b auto data_directory = std::make_unique(datadir_path); auto blocks_repository = db::blocks::make_blocks_repository(data_directory->snapshots().path(), /*open=*/false); auto state_repository = db::state::make_state_repository(data_directory->snapshots().path(), /*open=*/false); - db::DataStoreRef data_store{db::RWAccess{env}, blocks_repository, state_repository}; + db::DataStoreRef data_store{sw_mdbx::RWAccess{env}, blocks_repository, state_repository}; db::DataModelFactory data_model_factory{data_store}; // We need full snapshot sync to take place to have database tables properly updated @@ -300,7 +301,7 @@ void debug_unwind(db::EnvConfig& config, BlockNum height, uint32_t step, const b .stop_freezer = true, // do not generate new snapshots .no_seeding = true, // do not seed existing snapshots }; - struct EmptyStageScheduler : public stagedsync::StageScheduler { + struct EmptyStageScheduler : public datastore::StageScheduler { Task schedule(std::function /*callback*/) override { co_return; } }; EmptyStageScheduler empty_scheduler; @@ -324,7 +325,7 @@ void debug_unwind(db::EnvConfig& config, BlockNum height, uint32_t step, const b snap_sync_future.get(); // Commit is enabled by default in RWTxn(Managed), so we need to check here - db::RWTxnManaged txn{env}; + RWTxnManaged txn{env}; if (dry) { txn.disable_commit(); } else { @@ -381,13 +382,13 @@ void debug_unwind(db::EnvConfig& config, BlockNum height, uint32_t step, const b txn.commit_and_stop(); } -void unwind(db::EnvConfig& config, BlockNum unwind_point, const bool remove_blocks, const bool dry) { +void unwind(sw_mdbx::EnvConfig& config, BlockNum unwind_point, const bool remove_blocks, const bool dry) { ensure(config.exclusive, "Function requires exclusive access to database"); config.readonly = false; - auto env{silkworm::db::open_env(config)}; - db::RWTxnManaged txn{env}; + auto env{silkworm::sw_mdbx::open_env(config)}; + RWTxnManaged txn{env}; // Commit is enabled by default in RWTxn(Managed), so we need to check here if (dry) { @@ -405,7 +406,7 @@ void unwind(db::EnvConfig& config, BlockNum unwind_point, const bool remove_bloc auto data_directory = std::make_unique(); auto blocks_repository = db::blocks::make_blocks_repository(data_directory->snapshots().path(), /*open=*/true); auto state_repository = db::state::make_state_repository(data_directory->path(), /*open=*/true); - db::DataStoreRef data_store{db::RWAccess{env}, blocks_repository, state_repository}; + db::DataStoreRef data_store{sw_mdbx::RWAccess{env}, blocks_repository, state_repository}; db::DataModelFactory data_model_factory{data_store}; boost::asio::io_context io_context; @@ -448,7 +449,7 @@ void unwind(db::EnvConfig& config, BlockNum unwind_point, const bool remove_bloc const auto body_cursor{txn.rw_cursor(db::table::kBlockBodies)}; const auto start_key{db::block_key(unwind_point)}; std::size_t erased_bodies{0}; - auto body_data{body_cursor->lower_bound(db::to_slice(start_key), /*throw_notfound=*/false)}; + auto body_data{body_cursor->lower_bound(sw_mdbx::to_slice(start_key), /*throw_notfound=*/false)}; while (body_data) { body_cursor->erase(); ++erased_bodies; @@ -459,7 +460,7 @@ void unwind(db::EnvConfig& config, BlockNum unwind_point, const bool remove_bloc // Remove the block headers up to the unwind point const auto header_cursor{txn.rw_cursor(db::table::kHeaders)}; std::size_t erased_headers{0}; - auto header_data{header_cursor->lower_bound(db::to_slice(start_key), /*throw_notfound=*/false)}; + auto header_data{header_cursor->lower_bound(sw_mdbx::to_slice(start_key), /*throw_notfound=*/false)}; while (header_data) { header_cursor->erase(); ++erased_headers; @@ -470,7 +471,7 @@ void unwind(db::EnvConfig& config, BlockNum unwind_point, const bool remove_bloc // Remove the canonical hashes up to the unwind point const auto canonical_cursor{txn.rw_cursor(db::table::kCanonicalHashes)}; std::size_t erased_hashes{0}; - auto hash_data{canonical_cursor->lower_bound(db::to_slice(start_key), /*throw_notfound=*/false)}; + auto hash_data{canonical_cursor->lower_bound(sw_mdbx::to_slice(start_key), /*throw_notfound=*/false)}; while (hash_data) { canonical_cursor->erase(); ++erased_hashes; @@ -482,7 +483,7 @@ void unwind(db::EnvConfig& config, BlockNum unwind_point, const bool remove_bloc } } -void forward(db::EnvConfig& config, BlockNum forward_point, const bool dry, +void forward(sw_mdbx::EnvConfig& config, BlockNum forward_point, const bool dry, const std::string& start_at_stage, const std::string& stop_before_stage) { ensure(config.exclusive, "Function requires exclusive access to database"); @@ -491,8 +492,8 @@ void forward(db::EnvConfig& config, BlockNum forward_point, const bool dry, Environment::set_start_at_stage(start_at_stage); Environment::set_stop_before_stage(stop_before_stage); - auto env = silkworm::db::open_env(config); - db::RWTxnManaged txn{env}; + auto env = silkworm::sw_mdbx::open_env(config); + RWTxnManaged txn{env}; // Commit is enabled by default in RWTxn(Managed), so we need to check here if (dry) { @@ -513,7 +514,7 @@ void forward(db::EnvConfig& config, BlockNum forward_point, const bool dry, auto data_directory = std::make_unique(); auto blocks_repository = db::blocks::make_blocks_repository(data_directory->snapshots().path(), /*open=*/true); auto state_repository = db::state::make_state_repository(data_directory->path(), /*open=*/true); - db::DataStoreRef data_store{db::RWAccess{env}, blocks_repository, state_repository}; + db::DataStoreRef data_store{sw_mdbx::RWAccess{env}, blocks_repository, state_repository}; db::DataModelFactory data_model_factory{data_store}; boost::asio::io_context io_context; @@ -549,7 +550,7 @@ void forward(db::EnvConfig& config, BlockNum forward_point, const bool dry, std::cout << "\n Staged pipeline forward up to block: " << forward_point << " completed\n"; } -void bisect_pipeline(db::EnvConfig& config, BlockNum start, BlockNum end, const bool dry, +void bisect_pipeline(sw_mdbx::EnvConfig& config, BlockNum start, BlockNum end, const bool dry, const std::string& start_at_stage, const std::string& stop_before_stage) { ensure(config.exclusive, "Function requires exclusive access to database"); ensure(start > 0, "Function requires non-zero start block"); @@ -561,8 +562,8 @@ void bisect_pipeline(db::EnvConfig& config, BlockNum start, BlockNum end, const Environment::set_start_at_stage(start_at_stage); Environment::set_stop_before_stage(stop_before_stage); - auto env = silkworm::db::open_env(config); - db::RWTxnManaged txn{env}; + auto env = silkworm::sw_mdbx::open_env(config); + RWTxnManaged txn{env}; // Commit is enabled by default in RWTxn(Managed), so we need to check here if (dry) { @@ -583,7 +584,7 @@ void bisect_pipeline(db::EnvConfig& config, BlockNum start, BlockNum end, const auto data_directory = std::make_unique(); auto blocks_repository = db::blocks::make_blocks_repository(data_directory->snapshots().path(), /*open=*/true); auto state_repository = db::state::make_state_repository(data_directory->path(), /*open=*/true); - db::DataStoreRef data_store{db::RWAccess{env}, blocks_repository, state_repository}; + db::DataStoreRef data_store{sw_mdbx::RWAccess{env}, blocks_repository, state_repository}; db::DataModelFactory data_model_factory{data_store}; boost::asio::io_context io_context; @@ -650,7 +651,7 @@ void bisect_pipeline(db::EnvConfig& config, BlockNum start, BlockNum end, const } } -void reset_to_download(db::EnvConfig& config, const bool keep_senders, const bool force) { +void reset_to_download(sw_mdbx::EnvConfig& config, const bool keep_senders, const bool force) { if (!config.exclusive) { throw std::runtime_error("Function requires exclusive access to database"); } @@ -660,8 +661,8 @@ void reset_to_download(db::EnvConfig& config, const bool keep_senders, const boo return; } - auto env{silkworm::db::open_env(config)}; - db::RWTxnManaged txn(env); + auto env{silkworm::sw_mdbx::open_env(config)}; + RWTxnManaged txn(env); StopWatch sw(/*auto_start=*/true); // Void finish stage @@ -672,7 +673,7 @@ void reset_to_download(db::EnvConfig& config, const bool keep_senders, const boo // Void TxLookup stage SILK_INFO_M(db::stages::kTxLookupKey, {"table", db::table::kTxLookup.name}) << "truncating ..."; - db::PooledCursor source(*txn, db::table::kTxLookup); + PooledCursor source(*txn, db::table::kTxLookup); txn->clear_map(source.map()); db::stages::write_stage_progress(txn, db::stages::kTxLookupKey, 0); db::stages::write_stage_prune_progress(txn, db::stages::kTxLookupKey, 0); @@ -816,14 +817,14 @@ void reset_to_download(db::EnvConfig& config, const bool keep_senders, const boo SILK_INFO_M("All done", {"in", StopWatch::format(duration)}); } -void trie_account_analysis(db::EnvConfig& config) { +void trie_account_analysis(sw_mdbx::EnvConfig& config) { static std::string fmt_hdr{" %-24s %=50s "}; if (!config.exclusive) { throw std::runtime_error("Function requires exclusive access to database"); } - auto env{db::open_env(config)}; + auto env{sw_mdbx::open_env(config)}; auto txn{env.start_read()}; std::cout << "\n" @@ -832,7 +833,7 @@ void trie_account_analysis(db::EnvConfig& config) { << (boost::format(" %-24s ") % db::table::kTrieOfAccounts.name) << std::flush; std::map histogram; - auto code_cursor{db::open_cursor(txn, db::table::kTrieOfAccounts)}; + auto code_cursor = open_cursor(txn, db::table::kTrieOfAccounts); Progress progress{50}; size_t total_entries{txn.get_map_stat(code_cursor.map()).ms_entries}; @@ -863,7 +864,7 @@ void trie_account_analysis(db::EnvConfig& config) { std::cout << "\n\n"; } -void trie_scan(db::EnvConfig& config, bool del) { +void trie_scan(sw_mdbx::EnvConfig& config, bool del) { auto env{open_env(config)}; auto txn{env.start_write()}; std::vector tables{db::table::kTrieOfAccounts, db::table::kTrieOfStorage}; @@ -873,12 +874,12 @@ void trie_scan(db::EnvConfig& config, bool del) { if (SignalHandler::signalled()) { break; } - db::PooledCursor cursor(txn, map_config); + sw_mdbx::PooledCursor cursor(txn, map_config); std::cout << " Scanning " << map_config.name << "\n"; auto data{cursor.to_first(false)}; while (data) { if (data.value.empty()) { - std::cout << "Empty value at key " << to_hex(db::from_slice(data.key), true) << "\n"; + std::cout << "Empty value at key " << to_hex(sw_mdbx::from_slice(data.key), true) << "\n"; if (del) { cursor.erase(); } @@ -898,7 +899,7 @@ void trie_scan(db::EnvConfig& config, bool del) { std::cout << "\n\n"; } -void trie_integrity(db::EnvConfig& config, bool with_state_coverage, bool continue_scan, bool sanitize) { +void trie_integrity(sw_mdbx::EnvConfig& config, bool with_state_coverage, bool continue_scan, bool sanitize) { if (!config.exclusive) { throw std::runtime_error("Function requires exclusive access to database"); } @@ -912,9 +913,9 @@ void trie_integrity(db::EnvConfig& config, bool with_state_coverage, bool contin std::string source{db::table::kTrieOfAccounts.name}; bool is_healthy{true}; - db::PooledCursor trie_cursor1(txn, db::table::kTrieOfAccounts); - db::PooledCursor trie_cursor2(txn, db::table::kTrieOfAccounts); - db::PooledCursor state_cursor(txn, db::table::kHashedAccounts); + PooledCursor trie_cursor1{txn, db::table::kTrieOfAccounts}; + PooledCursor trie_cursor2{txn, db::table::kTrieOfAccounts}; + PooledCursor state_cursor{txn, db::table::kHashedAccounts}; size_t prefix_len{0}; Bytes buffer; @@ -935,8 +936,8 @@ void trie_integrity(db::EnvConfig& config, bool with_state_coverage, bool contin auto data1{trie_cursor1.to_first(false)}; while (data1) { - auto data1_k{db::from_slice(data1.key)}; - auto data1_v{db::from_slice(data1.value)}; + auto data1_k{sw_mdbx::from_slice(data1.key)}; + auto data1_v{sw_mdbx::from_slice(data1.value)}; auto node_k{data1_k.substr(prefix_len)}; // Only unmarshal relevant data without copy on read @@ -1030,14 +1031,14 @@ void trie_integrity(db::EnvConfig& config, bool with_state_coverage, bool contin continue; } buffer.back() = static_cast(i); - auto data2{trie_cursor2.lower_bound(db::to_slice(buffer), false)}; + auto data2{trie_cursor2.lower_bound(sw_mdbx::to_slice(buffer), false)}; if (!data2) { throw std::runtime_error("At key " + to_hex(data1_k, true) + " tree mask is " + std::bitset<16>(node_tree_mask).to_string() + " but there is no child " + std::to_string(i) + " in db. LTE found is : null"); } - auto data2_k{db::from_slice(data2.key)}; + auto data2_k{sw_mdbx::from_slice(data2.key)}; if (!data2_k.starts_with(buffer)) { throw std::runtime_error("At key " + to_hex(data1_k, true) + " tree mask is " + std::bitset<16>(node_tree_mask).to_string() + @@ -1061,12 +1062,12 @@ void trie_integrity(db::EnvConfig& config, bool with_state_coverage, bool contin for (size_t i{data1_k.size() - 1}; i >= prefix_len && !found; --i) { auto parent_seek_key{data1_k.substr(0, i)}; - auto data2{trie_cursor2.find(db::to_slice(parent_seek_key), false)}; + auto data2{trie_cursor2.find(sw_mdbx::to_slice(parent_seek_key), false)}; if (!data2) { continue; } found = true; - const auto data2_v{db::from_slice(data2.value)}; + const auto data2_v{sw_mdbx::from_slice(data2.value)}; const auto parent_tree_mask{endian::load_big_u16(&data2_v[2])}; const auto parent_child_id{static_cast(data1_k[i])}; const auto parent_has_tree_bit{(parent_tree_mask & (1 << parent_child_id)) != 0}; @@ -1145,9 +1146,9 @@ void trie_integrity(db::EnvConfig& config, bool with_state_coverage, bool contin // On first loop we search HashedAccounts (which is not dup-sorted) if (!loop_id) { - auto data3{state_cursor.lower_bound(db::to_slice(seek), false)}; + auto data3{state_cursor.lower_bound(sw_mdbx::to_slice(seek), false)}; if (data3) { - auto data3_k{db::from_slice(data3.key)}; + auto data3_k{sw_mdbx::from_slice(data3.key)}; if (data3_k.length() >= fixed_bytes) { found = (bits_to_match == 0 || ((data3_k.substr(0, fixed_bytes - 1) == seek.substr(0, fixed_bytes - 1)) && @@ -1159,17 +1160,17 @@ void trie_integrity(db::EnvConfig& config, bool with_state_coverage, bool contin std::bitset<16>(node_state_mask).to_string() + " but there is no child " + std::to_string(i) + "," + to_hex(seek, true) + " in hashed state"}; if (data3) { - auto data3_k{db::from_slice(data3.key)}; + auto data3_k{sw_mdbx::from_slice(data3.key)}; what.append(" found instead " + to_hex(data3_k, true)); } throw std::runtime_error(what); } } else { // On second loop we search HashedStorage (which is dup-sorted) - auto data3{state_cursor.lower_bound_multivalue(db::to_slice(data1_k.substr(0, prefix_len)), - db::to_slice(seek), false)}; + auto data3{state_cursor.lower_bound_multivalue(sw_mdbx::to_slice(data1_k.substr(0, prefix_len)), + sw_mdbx::to_slice(seek), false)}; if (data3) { - auto data3_v{db::from_slice(data3.value)}; + auto data3_v{sw_mdbx::from_slice(data3.value)}; if (data3_v.length() >= fixed_bytes) { found = (bits_to_match == 0 || ((data3_v.substr(0, fixed_bytes - 1) == seek.substr(0, fixed_bytes - 1)) && @@ -1181,8 +1182,8 @@ void trie_integrity(db::EnvConfig& config, bool with_state_coverage, bool contin std::bitset<16>(node_state_mask).to_string() + " but there is no child " + std::to_string(i) + "," + to_hex(seek, true) + " in state"}; if (data3) { - auto data3_k{db::from_slice(data3.key)}; - auto data3_v{db::from_slice(data3.value)}; + auto data3_k{sw_mdbx::from_slice(data3.key)}; + auto data3_v{sw_mdbx::from_slice(data3.value)}; what.append(" found instead " + to_hex(data3_k, true) + to_hex(data3_v, false)); } throw std::runtime_error(what); @@ -1213,7 +1214,7 @@ void trie_integrity(db::EnvConfig& config, bool with_state_coverage, bool contin env.close(); } -void trie_reset(db::EnvConfig& config, bool always_yes) { +void trie_reset(sw_mdbx::EnvConfig& config, bool always_yes) { if (!config.exclusive) { throw std::runtime_error("Function requires exclusive access to database"); } @@ -1225,7 +1226,7 @@ void trie_reset(db::EnvConfig& config, bool always_yes) { } auto env{open_env(config)}; - db::RWTxnManaged txn{env}; + RWTxnManaged txn{env}; SILK_INFO << "Clearing ..." << log::Args{"table", db::table::kTrieOfAccounts.name}; txn->clear_map(db::table::kTrieOfAccounts.name); SILK_INFO << "Clearing ..." << log::Args{"table", db::table::kTrieOfStorage.name}; @@ -1238,14 +1239,14 @@ void trie_reset(db::EnvConfig& config, bool always_yes) { env.close(); } -void trie_root(db::EnvConfig& config) { +void trie_root(sw_mdbx::EnvConfig& config) { if (!config.exclusive) { throw std::runtime_error("Function requires exclusive access to database"); } auto env{open_env(config)}; - db::ROTxnManaged txn{env}; - db::PooledCursor trie_accounts(txn, db::table::kTrieOfAccounts); + sw_mdbx::ROTxnManaged txn{env}; + PooledCursor trie_accounts(txn, db::table::kTrieOfAccounts); // Retrieve expected state root auto hashstate_stage_progress{db::stages::read_stage_progress(txn, db::stages::kHashStateKey)}; @@ -1253,8 +1254,8 @@ void trie_root(db::EnvConfig& config) { if (hashstate_stage_progress != intermediate_hashes_stage_progress) { throw std::runtime_error("HashState and Intermediate hashes stage progresses do not match"); } - auto header_hash{read_canonical_header_hash(txn, hashstate_stage_progress)}; - auto header{read_header(txn, hashstate_stage_progress, header_hash->bytes)}; + auto header_hash{db::read_canonical_header_hash(txn, hashstate_stage_progress)}; + auto header{db::read_header(txn, hashstate_stage_progress, header_hash->bytes)}; auto expected_state_root{header->state_root}; trie::PrefixSet empty_changes{}; // We need this to tell we have no changes. If nullptr means full regen @@ -1416,7 +1417,7 @@ int main(int argc, char* argv[]) { log::init(log_settings); DataDirectory data_dir{data_dir_factory()}; - db::EnvConfig chaindata_env_config{data_dir.chaindata().path().string()}; + sw_mdbx::EnvConfig chaindata_env_config{data_dir.chaindata().path().string()}; chaindata_env_config.shared = shared_opt->as(); chaindata_env_config.exclusive = exclusive_opt->as(); @@ -1428,10 +1429,10 @@ int main(int argc, char* argv[]) { .init_if_empty = true, }); } - const auto mdbx_path{db::get_datafile_path(data_dir.chaindata().path())}; + const auto mdbx_path{sw_mdbx::get_datafile_path(data_dir.chaindata().path())}; if (!fs::exists(mdbx_path) || !fs::is_regular_file(mdbx_path)) { std::cerr << "\n Directory " << data_dir.chaindata().path().string() << " does not contain " - << db::kDbDataFileName << "\n"; + << sw_mdbx::kDbDataFileName << "\n"; return -1; } diff --git a/cmd/silkworm.cpp b/cmd/silkworm.cpp index 9e8e2d98c7..f53a8ca179 100644 --- a/cmd/silkworm.cpp +++ b/cmd/silkworm.cpp @@ -152,7 +152,7 @@ void parse_silkworm_command_line(CLI::App& cli, int argc, char* argv[], node::Se throw std::invalid_argument("--chaindata.pagesize is not a power of 2"); } - const size_t mdbx_max_size_hard_limit = chaindata_page_size * db::kMdbxMaxPages; + const size_t mdbx_max_size_hard_limit = chaindata_page_size * sw_mdbx::kMdbxMaxPages; if (node_settings.chaindata_env_config.max_size > mdbx_max_size_hard_limit) { throw std::invalid_argument("--chaindata.maxsize exceeds max allowed size by page size i.e" + human_size(mdbx_max_size_hard_limit)); diff --git a/silkworm/capi/fork_validator.cpp b/silkworm/capi/fork_validator.cpp index 57babf27d7..1bc4ec81ed 100644 --- a/silkworm/capi/fork_validator.cpp +++ b/silkworm/capi/fork_validator.cpp @@ -25,9 +25,9 @@ #include "silkworm.h" static void set_node_settings(SilkwormHandle handle, const struct SilkwormForkValidatorSettings& settings, MDBX_env* mdbx_env) { - silkworm::db::EnvUnmanaged unmanaged_env{mdbx_env}; + silkworm::sw_mdbx::EnvUnmanaged unmanaged_env{mdbx_env}; - auto txn = silkworm::db::ROTxnManaged{unmanaged_env}; + auto txn = silkworm::sw_mdbx::ROTxnManaged{unmanaged_env}; auto chain_config{silkworm::db::read_chain_config(txn)}; SILKWORM_ASSERT(chain_config); @@ -43,7 +43,7 @@ static void set_node_settings(SilkwormHandle handle, const struct SilkwormForkVa handle->node_settings.data_directory = std::move(data_dir); auto db_env_flags = unmanaged_env.get_flags(); - handle->node_settings.chaindata_env_config = silkworm::db::EnvConfig{ + handle->node_settings.chaindata_env_config = silkworm::sw_mdbx::EnvConfig{ .path = handle->data_dir_path.string(), .create = false, .readonly = (db_env_flags & MDBX_RDONLY) != 0, @@ -124,8 +124,8 @@ SILKWORM_EXPORT int silkworm_start_fork_validator(SilkwormHandle handle, MDBX_en SILK_INFO << "Starting fork validator"; set_node_settings(handle, *settings, mdbx_env); - silkworm::db::EnvUnmanaged unmanaged_env{mdbx_env}; - silkworm::db::RWAccess rw_access{unmanaged_env}; + silkworm::sw_mdbx::EnvUnmanaged unmanaged_env{mdbx_env}; + silkworm::sw_mdbx::RWAccess rw_access{unmanaged_env}; silkworm::db::DataStoreRef data_store{ rw_access, *handle->blocks_repository, diff --git a/silkworm/capi/rpcdaemon.cpp b/silkworm/capi/rpcdaemon.cpp index 57b083bd2a..dd02332628 100644 --- a/silkworm/capi/rpcdaemon.cpp +++ b/silkworm/capi/rpcdaemon.cpp @@ -102,7 +102,7 @@ SILKWORM_EXPORT int silkworm_start_rpcdaemon(SilkwormHandle handle, MDBX_env* en auto daemon_settings = make_daemon_settings(handle, *settings); db::DataStoreRef data_store{ - db::RWAccess{db::EnvUnmanaged{env}}, + sw_mdbx::RWAccess{sw_mdbx::EnvUnmanaged{env}}, *handle->blocks_repository, *handle->state_repository, }; diff --git a/silkworm/capi/silkworm.cpp b/silkworm/capi/silkworm.cpp index 52ffeb3b59..1d2400cf7d 100644 --- a/silkworm/capi/silkworm.cpp +++ b/silkworm/capi/silkworm.cpp @@ -415,7 +415,7 @@ class BlockProvider { public: BlockProvider(BoundedBuffer>* block_buffer, - db::ROAccess db_access, + sw_mdbx::ROAccess db_access, db::DataModelFactory data_model_factory, BlockNum start_block, BlockNum max_block) : block_buffer_{block_buffer}, @@ -425,7 +425,7 @@ class BlockProvider { max_block_{max_block} {} void operator()() { - db::ROTxnManaged txn = db_access_.start_ro_tx(); + auto txn = db_access_.start_ro_tx(); db::DataModel access_layer = data_model_factory_(txn); BlockNum current_block{start_block_}; @@ -459,7 +459,7 @@ class BlockProvider { private: BoundedBuffer>* block_buffer_; - db::ROAccess db_access_; + sw_mdbx::ROAccess db_access_; db::DataModelFactory data_model_factory_; BlockNum start_block_; BlockNum max_block_; @@ -498,7 +498,7 @@ int silkworm_execute_blocks_ephemeral(SilkwormHandle handle, MDBX_txn* mdbx_txn, SignalHandlerGuard signal_guard; try { - auto txn = db::RWTxnUnmanaged{mdbx_txn}; + auto txn = sw_mdbx::RWTxnUnmanaged{mdbx_txn}; db::Buffer state_buffer{txn, std::make_unique(db::DataModel{txn, *handle->blocks_repository})}; state_buffer.set_memory_limit(batch_size); @@ -615,8 +615,8 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, try { // Wrap MDBX env into an internal *unmanaged* env, i.e. MDBX env is only used but its lifecycle is untouched - db::EnvUnmanaged unmanaged_env{mdbx_env}; - db::RWAccess rw_access{unmanaged_env}; + sw_mdbx::EnvUnmanaged unmanaged_env{mdbx_env}; + sw_mdbx::RWAccess rw_access{unmanaged_env}; auto txn = rw_access.start_rw_tx(); const auto env_path = unmanaged_env.get_path(); @@ -635,7 +635,7 @@ int silkworm_execute_blocks_perpetual(SilkwormHandle handle, MDBX_env* mdbx_env, BlockProvider block_provider{ &block_buffer, - db::ROAccess{unmanaged_env}, + sw_mdbx::ROAccess{unmanaged_env}, std::move(data_model_factory), start_block, max_block, diff --git a/silkworm/capi/silkworm_test.cpp b/silkworm/capi/silkworm_test.cpp index a4f577e113..de3cc63305 100644 --- a/silkworm/capi/silkworm_test.cpp +++ b/silkworm/capi/silkworm_test.cpp @@ -41,6 +41,7 @@ namespace silkworm { namespace snapshot_test = snapshots::test_util; using namespace silkworm::db; +using namespace silkworm::sw_mdbx; struct CApiTest { TemporaryDirectory tmp_dir; diff --git a/silkworm/db/CMakeLists.txt b/silkworm/db/CMakeLists.txt index 11c712ecb2..c3221a74ef 100644 --- a/silkworm/db/CMakeLists.txt +++ b/silkworm/db/CMakeLists.txt @@ -47,7 +47,7 @@ set(LIBS_PRIVATE Boost::headers gRPC::grpc++ protobuf::libprotobuf - silkworm_db_etl + silkworm_etl ) silkworm_library( diff --git a/silkworm/db/access_layer.cpp b/silkworm/db/access_layer.cpp index dcff9e4963..da4b8a700b 100644 --- a/silkworm/db/access_layer.cpp +++ b/silkworm/db/access_layer.cpp @@ -35,7 +35,9 @@ namespace silkworm::db { +using namespace sw_mdbx; using namespace snapshots; +using sw_mdbx::to_slice; std::optional read_schema_version(ROTxn& txn) { auto cursor = txn.ro_cursor(table::kDatabaseInfo); diff --git a/silkworm/db/access_layer.hpp b/silkworm/db/access_layer.hpp index 4e224fb6d3..d824e7d71f 100644 --- a/silkworm/db/access_layer.hpp +++ b/silkworm/db/access_layer.hpp @@ -44,6 +44,9 @@ class SnapshotRepository; namespace silkworm::db { +using sw_mdbx::ROTxn; +using sw_mdbx::RWTxn; + //! \brief Pulls database schema version std::optional read_schema_version(ROTxn& txn); @@ -184,7 +187,7 @@ void write_receipts(RWTxn& txn, const std::vector& receipts, // See Erigon ReadTransactions void read_transactions(ROTxn& txn, uint64_t base_id, uint64_t count, std::vector& out); -void read_transactions(ROCursor& txn_table, uint64_t base_id, uint64_t count, std::vector& out); +void read_transactions(sw_mdbx::ROCursor& txn_table, uint64_t base_id, uint64_t count, std::vector& out); bool read_rlp_transactions(ROTxn& txn, BlockNum block_num, const evmc::bytes32& hash, std::vector& rlp_txs); diff --git a/silkworm/db/access_layer_test.cpp b/silkworm/db/access_layer_test.cpp index f001b32e80..34a589a471 100644 --- a/silkworm/db/access_layer_test.cpp +++ b/silkworm/db/access_layer_test.cpp @@ -114,6 +114,9 @@ static BlockBody block_body_17035047() { namespace silkworm::db { +using namespace sw_mdbx; +using sw_mdbx::to_slice; + TEST_CASE("Methods cursor_for_each/cursor_for_count", "[db][access_layer]") { test_util::TempChainData context; auto& txn{context.rw_txn()}; diff --git a/silkworm/db/bitmap_test.cpp b/silkworm/db/bitmap_test.cpp index e5ce7b1ff2..01a2212c6c 100644 --- a/silkworm/db/bitmap_test.cpp +++ b/silkworm/db/bitmap_test.cpp @@ -24,7 +24,9 @@ #include #include -namespace silkworm::db::bitmap { +namespace silkworm::sw_mdbx::bitmap { + +using namespace silkworm::db; static void cut_everything(roaring::Roaring& bm, uint64_t limit) { while (bm.cardinality() > 0) { @@ -127,7 +129,7 @@ TEST_CASE("Bitmap Index Loader") { {Bytes(address3.bytes, kAddressLength), roaring3}, }; - etl_mdbx::Collector collector(context.dir().temp().path()); + sw_mdbx::Collector collector(context.dir().temp().path()); IndexLoader bm_loader(table::kLogAddressIndex); IndexLoader::flush_bitmaps_to_etl(bitmaps, &collector, /*flush_count=*/1); REQUIRE(collector.bytes_size()); @@ -211,4 +213,4 @@ TEST_CASE("Bitmap Index Loader") { REQUIRE(bm_loader.get_current_key().empty()); } -} // namespace silkworm::db::bitmap +} // namespace silkworm::sw_mdbx::bitmap diff --git a/silkworm/db/blocks/bodies/body_segment_collation.cpp b/silkworm/db/blocks/bodies/body_segment_collation.cpp index b88c328253..3bdac16a04 100644 --- a/silkworm/db/blocks/bodies/body_segment_collation.cpp +++ b/silkworm/db/blocks/bodies/body_segment_collation.cpp @@ -25,6 +25,9 @@ namespace silkworm::db { +using namespace sw_mdbx; +using namespace datastore; + void BodySegmentCollation::copy(ROTxn& txn, const SegmentCollationCommand& command, snapshots::segment::SegmentFileWriter& file_writer) const { BlockNumRange range = command.range; uint64_t base_txn_id = command.base_txn_id; diff --git a/silkworm/db/blocks/bodies/body_segment_collation.hpp b/silkworm/db/blocks/bodies/body_segment_collation.hpp index 579f6394b1..09b86ac1f7 100644 --- a/silkworm/db/blocks/bodies/body_segment_collation.hpp +++ b/silkworm/db/blocks/bodies/body_segment_collation.hpp @@ -20,11 +20,11 @@ namespace silkworm::db { -class BodySegmentCollation : public SegmentCollation { +class BodySegmentCollation : public datastore::SegmentCollation { public: ~BodySegmentCollation() override = default; - void copy(ROTxn& txn, const SegmentCollationCommand& command, snapshots::segment::SegmentFileWriter& file_writer) const override; - void prune(RWTxn& txn, BlockNumRange range) const override; + void copy(sw_mdbx::ROTxn& txn, const datastore::SegmentCollationCommand& command, snapshots::segment::SegmentFileWriter& file_writer) const override; + void prune(sw_mdbx::RWTxn& txn, BlockNumRange range) const override; }; } // namespace silkworm::db diff --git a/silkworm/db/blocks/headers/header_segment_collation.cpp b/silkworm/db/blocks/headers/header_segment_collation.cpp index 71e7e6571b..0fc55b294a 100644 --- a/silkworm/db/blocks/headers/header_segment_collation.cpp +++ b/silkworm/db/blocks/headers/header_segment_collation.cpp @@ -25,6 +25,9 @@ namespace silkworm::db { +using namespace sw_mdbx; +using namespace datastore; + void HeaderSegmentCollation::copy(ROTxn& txn, const SegmentCollationCommand& command, snapshots::segment::SegmentFileWriter& file_writer) const { BlockNumRange range = command.range; snapshots::HeaderSegmentWriter writer{file_writer}; diff --git a/silkworm/db/blocks/headers/header_segment_collation.hpp b/silkworm/db/blocks/headers/header_segment_collation.hpp index c713ebb8a7..2b55c10a43 100644 --- a/silkworm/db/blocks/headers/header_segment_collation.hpp +++ b/silkworm/db/blocks/headers/header_segment_collation.hpp @@ -20,11 +20,11 @@ namespace silkworm::db { -class HeaderSegmentCollation : public SegmentCollation { +class HeaderSegmentCollation : public datastore::SegmentCollation { public: ~HeaderSegmentCollation() override = default; - void copy(ROTxn& txn, const SegmentCollationCommand& command, snapshots::segment::SegmentFileWriter& file_writer) const override; - void prune(RWTxn& txn, BlockNumRange range) const override; + void copy(sw_mdbx::ROTxn& txn, const datastore::SegmentCollationCommand& command, snapshots::segment::SegmentFileWriter& file_writer) const override; + void prune(sw_mdbx::RWTxn& txn, BlockNumRange range) const override; }; } // namespace silkworm::db diff --git a/silkworm/db/blocks/transactions/txn_index.hpp b/silkworm/db/blocks/transactions/txn_index.hpp index 445e74edfe..03d1ce35bc 100644 --- a/silkworm/db/blocks/transactions/txn_index.hpp +++ b/silkworm/db/blocks/transactions/txn_index.hpp @@ -72,7 +72,7 @@ class TransactionIndex { .key_factory = std::make_unique(first_tx_id), .base_data_id = first_tx_id, .less_false_positives = true, - .etl_buffer_size = db::etl::kOptimalBufferSize / 2, + .etl_buffer_size = etl::kOptimalBufferSize / 2, }; } }; diff --git a/silkworm/db/blocks/transactions/txn_segment_collation.cpp b/silkworm/db/blocks/transactions/txn_segment_collation.cpp index fd53061018..894047a42d 100644 --- a/silkworm/db/blocks/transactions/txn_segment_collation.cpp +++ b/silkworm/db/blocks/transactions/txn_segment_collation.cpp @@ -25,6 +25,9 @@ namespace silkworm::db { +using namespace sw_mdbx; +using namespace datastore; + void TransactionSegmentCollation::copy(ROTxn& txn, const SegmentCollationCommand& command, snapshots::segment::SegmentFileWriter& file_writer) const { BlockNumRange range = command.range; snapshots::TransactionSegmentWriter writer{file_writer}; diff --git a/silkworm/db/blocks/transactions/txn_segment_collation.hpp b/silkworm/db/blocks/transactions/txn_segment_collation.hpp index f2cba219ad..02886fa92d 100644 --- a/silkworm/db/blocks/transactions/txn_segment_collation.hpp +++ b/silkworm/db/blocks/transactions/txn_segment_collation.hpp @@ -20,11 +20,11 @@ namespace silkworm::db { -class TransactionSegmentCollation : public SegmentCollation { +class TransactionSegmentCollation : public datastore::SegmentCollation { public: ~TransactionSegmentCollation() override = default; - void copy(ROTxn& txn, const SegmentCollationCommand& command, snapshots::segment::SegmentFileWriter& file_writer) const override; - void prune(RWTxn& txn, BlockNumRange range) const override; + void copy(sw_mdbx::ROTxn& txn, const datastore::SegmentCollationCommand& command, snapshots::segment::SegmentFileWriter& file_writer) const override; + void prune(sw_mdbx::RWTxn& txn, BlockNumRange range) const override; }; } // namespace silkworm::db diff --git a/silkworm/db/blocks/transactions/txn_to_block_index.hpp b/silkworm/db/blocks/transactions/txn_to_block_index.hpp index d428a6809f..b4ab00b5bc 100644 --- a/silkworm/db/blocks/transactions/txn_to_block_index.hpp +++ b/silkworm/db/blocks/transactions/txn_to_block_index.hpp @@ -100,7 +100,7 @@ class TransactionToBlockIndex { .key_factory = std::make_unique(first_tx_id), .base_data_id = first_block_num, .double_enum_index = false, - .etl_buffer_size = db::etl::kOptimalBufferSize / 2, + .etl_buffer_size = etl::kOptimalBufferSize / 2, }; } }; diff --git a/silkworm/db/buffer.cpp b/silkworm/db/buffer.cpp index 8ed06529cc..8f7f2e7b18 100644 --- a/silkworm/db/buffer.cpp +++ b/silkworm/db/buffer.cpp @@ -31,6 +31,8 @@ namespace silkworm::db { +using sw_mdbx::to_slice; + template size_t flat_hash_map_memory_size(size_t capacity) { return sizeof(std::pair) * capacity; diff --git a/silkworm/db/buffer_test.cpp b/silkworm/db/buffer_test.cpp index 836c9449dc..f4599152fa 100644 --- a/silkworm/db/buffer_test.cpp +++ b/silkworm/db/buffer_test.cpp @@ -239,6 +239,7 @@ TEST_CASE("Buffer storage", "[silkworm][db][buffer]") { } TEST_CASE("Buffer account", "[silkworm][db][buffer]") { + using sw_mdbx::from_slice; db::test_util::TempChainData context; auto& txn{context.rw_txn()}; diff --git a/silkworm/db/chain_data_init.cpp b/silkworm/db/chain_data_init.cpp index 4d2a4fb933..e56d40b15b 100644 --- a/silkworm/db/chain_data_init.cpp +++ b/silkworm/db/chain_data_init.cpp @@ -40,13 +40,13 @@ ChainConfig chain_data_init(const ChainDataInitSettings& node_settings) { }); auto chaindata_env_config = node_settings.chaindata_env_config; - chaindata_env_config.create = !std::filesystem::exists(get_datafile_path(chaindata_env_config.path)); + chaindata_env_config.create = !std::filesystem::exists(sw_mdbx::get_datafile_path(chaindata_env_config.path)); chaindata_env_config.exclusive = true; // Open chaindata environment and check tables are consistent log::Info("Opening database", {"path", chaindata_env_config.path}); mdbx::env_managed chaindata_env = open_env(chaindata_env_config); - RWTxnManaged tx(chaindata_env); + sw_mdbx::RWTxnManaged tx(chaindata_env); // Ensures all tables are present table::check_or_create_chaindata_tables(tx); diff --git a/silkworm/db/chain_data_init.hpp b/silkworm/db/chain_data_init.hpp index e5334f9c2e..95e2f75871 100644 --- a/silkworm/db/chain_data_init.hpp +++ b/silkworm/db/chain_data_init.hpp @@ -23,7 +23,7 @@ namespace silkworm::db { struct ChainDataInitSettings { - db::EnvConfig chaindata_env_config; + sw_mdbx::EnvConfig chaindata_env_config; db::PruneMode prune_mode; ChainId network_id{0}; bool init_if_empty{true}; diff --git a/silkworm/db/chain_head.cpp b/silkworm/db/chain_head.cpp index 8eb50ff78a..daf1d8b138 100644 --- a/silkworm/db/chain_head.cpp +++ b/silkworm/db/chain_head.cpp @@ -24,6 +24,8 @@ namespace silkworm::db { +using namespace sw_mdbx; + ChainHead read_chain_head(ROTxn& txn) { ChainHead chain_head; @@ -48,7 +50,7 @@ ChainHead read_chain_head(ROTxn& txn) { return chain_head; } -ChainHead read_chain_head(db::ROAccess db_access) { +ChainHead read_chain_head(sw_mdbx::ROAccess db_access) { auto txn = db_access.start_ro_tx(); [[maybe_unused]] auto _ = gsl::finally([&txn] { txn.abort(); }); diff --git a/silkworm/db/chain_head.hpp b/silkworm/db/chain_head.hpp index 782ae1debd..236534cadb 100644 --- a/silkworm/db/chain_head.hpp +++ b/silkworm/db/chain_head.hpp @@ -21,7 +21,7 @@ namespace silkworm::db { -ChainHead read_chain_head(ROTxn& txn); -ChainHead read_chain_head(db::ROAccess db_access); +ChainHead read_chain_head(sw_mdbx::ROTxn& txn); +ChainHead read_chain_head(sw_mdbx::ROAccess db_access); } // namespace silkworm::db diff --git a/silkworm/db/data_store.hpp b/silkworm/db/data_store.hpp index a0a0725703..c765992165 100644 --- a/silkworm/db/data_store.hpp +++ b/silkworm/db/data_store.hpp @@ -25,7 +25,7 @@ namespace silkworm::db { struct DataStoreRef { - RWAccess chaindata; + sw_mdbx::RWAccess chaindata; snapshots::SnapshotRepository& blocks_repository; snapshots::SnapshotRepository& state_repository; }; @@ -44,10 +44,10 @@ class DataStore { } {} DataStore( - const EnvConfig& chaindata_env_config, + const sw_mdbx::EnvConfig& chaindata_env_config, const std::filesystem::path& repository_path) : DataStore{ - db::open_env(chaindata_env_config), + sw_mdbx::open_env(chaindata_env_config), blocks::make_blocks_repository(repository_path), state::make_state_repository(repository_path), } {} @@ -64,8 +64,8 @@ class DataStore { }; } - db::ROAccess chaindata() const { return store_.chaindata(); } - db::RWAccess chaindata_rw() const { return store_.chaindata_rw(); } + sw_mdbx::ROAccess chaindata() const { return store_.chaindata(); } + sw_mdbx::RWAccess chaindata_rw() const { return store_.chaindata_rw(); } private: static datastore::Schema make_schema(); diff --git a/silkworm/db/datastore/CMakeLists.txt b/silkworm/db/datastore/CMakeLists.txt index 29006bc800..19653796c7 100644 --- a/silkworm/db/datastore/CMakeLists.txt +++ b/silkworm/db/datastore/CMakeLists.txt @@ -29,7 +29,7 @@ set(LIBS_PUBLIC silkworm_core silkworm_infra silkworm_datastore_common - silkworm_db_mdbx + silkworm_mdbx silkworm_snapshots ) # cmake-format: on diff --git a/silkworm/db/datastore/data_migration.cpp b/silkworm/db/datastore/data_migration.cpp index 096a3cff2f..c6a0faf49e 100644 --- a/silkworm/db/datastore/data_migration.cpp +++ b/silkworm/db/datastore/data_migration.cpp @@ -21,7 +21,7 @@ #include #include -namespace silkworm::db { +namespace silkworm::datastore { Task DataMigration::exec() { SILK_DEBUG_M(name()) << "START"; @@ -52,4 +52,4 @@ Task DataMigration::run_loop() { } } -} // namespace silkworm::db +} // namespace silkworm::datastore diff --git a/silkworm/db/datastore/data_migration.hpp b/silkworm/db/datastore/data_migration.hpp index b8d6591040..4a82b80da4 100644 --- a/silkworm/db/datastore/data_migration.hpp +++ b/silkworm/db/datastore/data_migration.hpp @@ -22,7 +22,7 @@ #include "data_migration_command.hpp" -namespace silkworm::db { +namespace silkworm::datastore { struct DataMigrationResult { virtual ~DataMigrationResult() = default; @@ -43,4 +43,4 @@ struct DataMigration { virtual Task cleanup() = 0; }; -} // namespace silkworm::db +} // namespace silkworm::datastore diff --git a/silkworm/db/datastore/data_migration_command.hpp b/silkworm/db/datastore/data_migration_command.hpp index f38eaab21a..84203a781f 100644 --- a/silkworm/db/datastore/data_migration_command.hpp +++ b/silkworm/db/datastore/data_migration_command.hpp @@ -18,11 +18,11 @@ #include -namespace silkworm::db { +namespace silkworm::datastore { struct DataMigrationCommand { virtual ~DataMigrationCommand() = default; virtual std::string description() const = 0; }; -} // namespace silkworm::db +} // namespace silkworm::datastore diff --git a/silkworm/db/datastore/data_store.hpp b/silkworm/db/datastore/data_store.hpp index 5b953c9a19..ad4e001705 100644 --- a/silkworm/db/datastore/data_store.hpp +++ b/silkworm/db/datastore/data_store.hpp @@ -43,8 +43,8 @@ class DataStore { } const Schema& schema() const { return schema_; } - db::ROAccess chaindata() const { return db::ROAccess{chaindata_env_}; } - db::RWAccess chaindata_rw() const { return db::RWAccess{chaindata_env_}; } + sw_mdbx::ROAccess chaindata() const { return sw_mdbx::ROAccess{chaindata_env_}; } + sw_mdbx::RWAccess chaindata_rw() const { return sw_mdbx::RWAccess{chaindata_env_}; } snapshots::SnapshotRepository& repository(const EntityName& name) const { return *repositories_.at(name); } diff --git a/silkworm/db/datastore/etl/CMakeLists.txt b/silkworm/db/datastore/etl/CMakeLists.txt index 502a03869e..5abf539d04 100644 --- a/silkworm/db/datastore/etl/CMakeLists.txt +++ b/silkworm/db/datastore/etl/CMakeLists.txt @@ -17,7 +17,7 @@ include("${SILKWORM_MAIN_DIR}/cmake/common/targets.cmake") silkworm_library( - silkworm_db_etl + silkworm_etl PUBLIC silkworm_core PRIVATE silkworm_infra ) diff --git a/silkworm/db/datastore/etl/buffer.hpp b/silkworm/db/datastore/etl/buffer.hpp index 32803cabd7..6e40eab20a 100644 --- a/silkworm/db/datastore/etl/buffer.hpp +++ b/silkworm/db/datastore/etl/buffer.hpp @@ -23,7 +23,7 @@ #include "util.hpp" -namespace silkworm::db::etl { +namespace silkworm::etl { inline constexpr size_t kInitialBufferCapacity = 32768; @@ -72,4 +72,4 @@ class Buffer { std::vector buffer_; // buffer for holding entries }; -} // namespace silkworm::db::etl +} // namespace silkworm::etl diff --git a/silkworm/db/datastore/etl/collector.cpp b/silkworm/db/datastore/etl/collector.cpp index de356c2de3..aba27dbbef 100644 --- a/silkworm/db/datastore/etl/collector.cpp +++ b/silkworm/db/datastore/etl/collector.cpp @@ -26,7 +26,7 @@ #include #include -namespace silkworm::db::etl { +namespace silkworm::etl { namespace fs = std::filesystem; @@ -181,4 +181,4 @@ std::filesystem::path Collector::set_work_path(const std::optional -namespace silkworm::db::etl { +namespace silkworm::etl { struct CollectorSettings { std::filesystem::path work_path; size_t buffer_size{}; }; -} // namespace silkworm::db::etl +} // namespace silkworm::etl diff --git a/silkworm/db/datastore/etl/file_provider.cpp b/silkworm/db/datastore/etl/file_provider.cpp index 90a587c97e..2225992a0d 100644 --- a/silkworm/db/datastore/etl/file_provider.cpp +++ b/silkworm/db/datastore/etl/file_provider.cpp @@ -21,7 +21,7 @@ #include #include -namespace silkworm::db::etl { +namespace silkworm::etl { namespace fs = std::filesystem; @@ -109,4 +109,4 @@ std::string FileProvider::get_file_name() const { return file_name_; } size_t FileProvider::get_file_size() const { return file_size_; } -} // namespace silkworm::db::etl +} // namespace silkworm::etl diff --git a/silkworm/db/datastore/etl/file_provider.hpp b/silkworm/db/datastore/etl/file_provider.hpp index 94c0478631..e89bf4f8ef 100644 --- a/silkworm/db/datastore/etl/file_provider.hpp +++ b/silkworm/db/datastore/etl/file_provider.hpp @@ -23,7 +23,7 @@ #include "buffer.hpp" #include "util.hpp" -namespace silkworm::db::etl { +namespace silkworm::etl { /** * Provides an abstraction to flush data to disk @@ -48,4 +48,4 @@ class FileProvider { size_t file_size_{0}; // Actual size of written data }; -} // namespace silkworm::db::etl +} // namespace silkworm::etl diff --git a/silkworm/db/datastore/etl/in_memory_collector.hpp b/silkworm/db/datastore/etl/in_memory_collector.hpp index 61b159ce7f..0a998983d3 100644 --- a/silkworm/db/datastore/etl/in_memory_collector.hpp +++ b/silkworm/db/datastore/etl/in_memory_collector.hpp @@ -36,7 +36,7 @@ * It can be used to prototype code that will use the full ETL collector or to do performance comparisons * between a memory-only impl. and a file-based impl. */ -namespace silkworm::db::etl { +namespace silkworm::etl { // Function pointer to process Load on before Load data into tables using KVLoadFunc = std::function; @@ -155,4 +155,4 @@ class InMemoryCollector { std::string loading_key_{}; // Actual load key (for log purposes) }; -} // namespace silkworm::db::etl +} // namespace silkworm::etl diff --git a/silkworm/db/datastore/etl/util.hpp b/silkworm/db/datastore/etl/util.hpp index e0650b41f6..185cb809d0 100644 --- a/silkworm/db/datastore/etl/util.hpp +++ b/silkworm/db/datastore/etl/util.hpp @@ -21,7 +21,7 @@ #include -namespace silkworm::db::etl { +namespace silkworm::etl { class EtlError : public std::runtime_error { public: @@ -58,4 +58,4 @@ inline bool operator<(const Entry& a, const Entry& b) { return diff < 0; } -} // namespace silkworm::db::etl +} // namespace silkworm::etl diff --git a/silkworm/db/datastore/etl/util_test.cpp b/silkworm/db/datastore/etl/util_test.cpp index 12aaa73346..6fca913651 100644 --- a/silkworm/db/datastore/etl/util_test.cpp +++ b/silkworm/db/datastore/etl/util_test.cpp @@ -20,7 +20,7 @@ #include -namespace silkworm::db::etl { +namespace silkworm::etl { TEST_CASE("ETL Entry comparison") { Entry a{*from_hex("1a"), *from_hex("75")}; @@ -38,4 +38,4 @@ TEST_CASE("ETL Entry comparison") { CHECK(!(d < d)); } -} // namespace silkworm::db::etl +} // namespace silkworm::etl diff --git a/silkworm/db/datastore/mdbx/CMakeLists.txt b/silkworm/db/datastore/mdbx/CMakeLists.txt index 0bd60b18de..37e60bb6df 100644 --- a/silkworm/db/datastore/mdbx/CMakeLists.txt +++ b/silkworm/db/datastore/mdbx/CMakeLists.txt @@ -20,14 +20,14 @@ find_package(absl REQUIRED) find_package(roaring REQUIRED) silkworm_library( - silkworm_db_mdbx + silkworm_mdbx PUBLIC absl::btree absl::function_ref roaring::roaring mdbx-static silkworm_core silkworm_infra - silkworm_db_etl + silkworm_etl ) -target_link_libraries(silkworm_db_mdbx_test PRIVATE silkworm_infra_test_util) +target_link_libraries(silkworm_mdbx_test PRIVATE silkworm_infra_test_util) diff --git a/silkworm/db/datastore/mdbx/bitmap.cpp b/silkworm/db/datastore/mdbx/bitmap.cpp index 0e8491cea1..43f229d398 100644 --- a/silkworm/db/datastore/mdbx/bitmap.cpp +++ b/silkworm/db/datastore/mdbx/bitmap.cpp @@ -26,7 +26,7 @@ #include "etl_mdbx_collector.hpp" -namespace silkworm::db::bitmap { +namespace silkworm::sw_mdbx::bitmap { template Bytes upper_bound_suffix(BlockUpperBound value) { @@ -89,7 +89,7 @@ RoaringMap cut_left_impl(RoaringMap& bm, uint64_t size_limit) { } template -void IndexLoader::merge_bitmaps_impl(RWTxn& txn, size_t key_size, etl_mdbx::Collector* bitmaps_collector) { +void IndexLoader::merge_bitmaps_impl(RWTxn& txn, size_t key_size, sw_mdbx::Collector* bitmaps_collector) { // Cannot use block_key because we need block number serialized in sizeof(BlockUpperBound) bytes Bytes last_shard_suffix{upper_bound_suffix(std::numeric_limits::max())}; @@ -97,11 +97,11 @@ void IndexLoader::merge_bitmaps_impl(RWTxn& txn, size_t key_size, etl_mdbx::Coll max_value_size_for_leaf_page(*txn, key_size + /*shard upper_bound*/ sizeof(BlockUpperBound))}; PooledCursor target(txn, index_config_); - etl_mdbx::LoadFunc load_func{[&last_shard_suffix, &optimal_shard_size]( - const etl::Entry& entry, - RWCursorDupSort& index_cursor, - MDBX_put_flags_t put_flags) -> void { - auto new_bitmap{bitmap::parse_impl(entry.value)}; // Bitmap being merged + sw_mdbx::LoadFunc load_func{[&last_shard_suffix, &optimal_shard_size]( + const etl::Entry& entry, + RWCursorDupSort& index_cursor, + MDBX_put_flags_t put_flags) -> void { + auto new_bitmap{parse_impl(entry.value)}; // Bitmap being merged // Check whether we have any previous shard to merge with Bytes shard_key{ @@ -111,17 +111,17 @@ void IndexLoader::merge_bitmaps_impl(RWTxn& txn, size_t key_size, etl_mdbx::Coll if (auto index_data{index_cursor.find(to_slice(shard_key), /*throw_notfound=*/false)}; index_data.done) { // Merge previous and current bitmap - new_bitmap |= bitmap::parse_impl(index_data.value); + new_bitmap |= parse_impl(index_data.value); index_cursor.erase(); // Delete currently found record as it'll be rewritten } // Consume bitmap splitting in shards while (!new_bitmap.isEmpty()) { - auto shard{bitmap::cut_left_impl(new_bitmap, optimal_shard_size)}; + auto shard{cut_left_impl(new_bitmap, optimal_shard_size)}; const bool consumed_to_last_chunk{new_bitmap.isEmpty()}; const BlockUpperBound suffix{consumed_to_last_chunk ? std::numeric_limits::max() : shard.maximum()}; intx::be::unsafe::store(&shard_key[shard_key.size() - sizeof(BlockUpperBound)], suffix); - Bytes shard_bytes{bitmap::to_bytes(shard)}; + Bytes shard_bytes{to_bytes(shard)}; mdbx::slice k{to_slice(shard_key)}; mdbx::slice v{to_slice(shard_bytes)}; mdbx::error::success_or_throw(index_cursor.put(k, &v, put_flags)); @@ -168,7 +168,7 @@ void IndexLoader::unwind_bitmaps_impl(RWTxn& txn, BlockNum to, const std::map(index_data.value)}; + auto db_bitmap{parse_impl(index_data.value)}; if (db_bitmap.maximum() <= to) { break; } @@ -186,7 +186,7 @@ void IndexLoader::unwind_bitmaps_impl(RWTxn& txn, BlockNum to, const std::map(txn, key_size, bitmaps_collector); } -void IndexLoader::merge_bitmaps(RWTxn& txn, size_t key_size, etl_mdbx::Collector* bitmaps_collector) { +void IndexLoader::merge_bitmaps(RWTxn& txn, size_t key_size, sw_mdbx::Collector* bitmaps_collector) { merge_bitmaps_impl(txn, key_size, bitmaps_collector); } @@ -240,7 +240,7 @@ void IndexLoader::prune_bitmaps_impl(RWTxn& txn, BlockNum threshold) { target.erase(); } else { // Read current bitmap - auto bitmap{bitmap::parse_impl(target_data.value)}; + auto bitmap{parse_impl(target_data.value)}; bool shard_shrunk{false}; while (!bitmap.isEmpty() && bitmap.minimum() <= threshold) { bitmap.remove(bitmap.minimum()); @@ -248,7 +248,7 @@ void IndexLoader::prune_bitmaps_impl(RWTxn& txn, BlockNum threshold) { } if (bitmap.isEmpty() || shard_shrunk) { if (!bitmap.isEmpty()) { - Bytes new_shard_data{bitmap::to_bytes(bitmap)}; + Bytes new_shard_data{to_bytes(bitmap)}; target.update(to_slice(data_key_view), to_slice(new_shard_data)); } else { target.erase(); @@ -277,7 +277,7 @@ void flush_bitmaps_impl(absl::btree_map& bitmaps, etl::Collec Bytes etl_key(key.size() + sizeof(uint16_t), '\0'); std::memcpy(&etl_key[0], key.data(), key.size()); endian::store_big_u16(&etl_key[key.size()], flush_count); - collector->collect({etl_key, bitmap::to_bytes(bitmap)}); + collector->collect({etl_key, to_bytes(bitmap)}); } bitmaps.clear(); } @@ -338,4 +338,4 @@ roaring::Roaring parse32(const mdbx::slice& data) { return parse_impl(data); } -} // namespace silkworm::db::bitmap +} // namespace silkworm::sw_mdbx::bitmap diff --git a/silkworm/db/datastore/mdbx/bitmap.hpp b/silkworm/db/datastore/mdbx/bitmap.hpp index 89e8f2ede2..d0bcc6b3fd 100644 --- a/silkworm/db/datastore/mdbx/bitmap.hpp +++ b/silkworm/db/datastore/mdbx/bitmap.hpp @@ -40,27 +40,27 @@ #include "mdbx.hpp" -namespace silkworm::db::etl { +namespace silkworm::etl { class Collector; } -namespace silkworm::db::etl_mdbx { +namespace silkworm::sw_mdbx { class Collector; } -namespace silkworm::db::bitmap { +namespace silkworm::sw_mdbx::bitmap { class IndexLoader { public: - explicit IndexLoader(const db::MapConfig& index_config) : index_config_{index_config} {} + explicit IndexLoader(const MapConfig& index_config) : index_config_{index_config} {} //! \brief Merges a list of bitmaps, previously collected, into index table ensuring //! all bitmaps are properly sharded and that last bitmap is marked with an UINT64_MAX upper bound //! \param txn [in] : An MDBX transaction holder //! \param key_size [in] : The actual length of key in the list of bitmap shards (the index) - //! \param collector [in] : A pointer to the etl::collector holding the bitmaps to be merged - void merge_bitmaps(RWTxn& txn, size_t key_size, etl_mdbx::Collector* bitmaps_collector); - void merge_bitmaps32(RWTxn& txn, size_t key_size, etl_mdbx::Collector* bitmaps_collector); + //! \param bitmaps_collector [in] : A pointer to the etl::collector holding the bitmaps to be merged + void merge_bitmaps(RWTxn& txn, size_t key_size, sw_mdbx::Collector* bitmaps_collector); + void merge_bitmaps32(RWTxn& txn, size_t key_size, sw_mdbx::Collector* bitmaps_collector); //! \brief Provided a list of keys for which the unwind should be applied this removes right values from shards //! \param txn [in] : An MDBX transaction holder @@ -95,7 +95,7 @@ class IndexLoader { private: template - void merge_bitmaps_impl(RWTxn& txn, size_t key_size, etl_mdbx::Collector* bitmaps_collector); + void merge_bitmaps_impl(RWTxn& txn, size_t key_size, sw_mdbx::Collector* bitmaps_collector); template void unwind_bitmaps_impl(RWTxn& txn, BlockNum to, const std::map& keys); @@ -103,9 +103,9 @@ class IndexLoader { template void prune_bitmaps_impl(RWTxn& txn, BlockNum threshold); - const db::MapConfig& index_config_; // The bucket config holding the index of maps - mutable std::mutex log_mtx_; // To get progress status - std::string current_key_; // Key being processed + const MapConfig& index_config_; // The bucket config holding the index of maps + mutable std::mutex log_mtx_; // To get progress status + std::string current_key_; // Key being processed }; // Return the first value in the bitmap that is not less than (i.e. greater or equal to) n, @@ -134,4 +134,4 @@ roaring::Roaring64Map parse(ByteView data); //! \brief Parse 32-bit roaring bitmap from MDBX slice roaring::Roaring parse32(const mdbx::slice& data); -} // namespace silkworm::db::bitmap +} // namespace silkworm::sw_mdbx::bitmap diff --git a/silkworm/db/datastore/mdbx/etl_mdbx_collector.hpp b/silkworm/db/datastore/mdbx/etl_mdbx_collector.hpp index a96c0cddaf..8ed09734d8 100644 --- a/silkworm/db/datastore/mdbx/etl_mdbx_collector.hpp +++ b/silkworm/db/datastore/mdbx/etl_mdbx_collector.hpp @@ -20,10 +20,10 @@ #include "mdbx.hpp" -namespace silkworm::db::etl_mdbx { +namespace silkworm::sw_mdbx { // Function pointer to process Load on before Load data into tables -using LoadFunc = std::function; +using LoadFunc = std::function; class Collector : public etl::Collector { public: @@ -34,18 +34,18 @@ class Collector : public etl::Collector { //! \param [in] load_func : Pointer to function transforming collected entries. If NULL no transform is executed //! \param [in] flags : Optional put flags for append or upsert (default) items void load( - db::RWCursorDupSort& target, + RWCursorDupSort& target, const LoadFunc& load_func = {}, MDBX_put_flags_t flags = MDBX_put_flags_t::MDBX_UPSERT) { etl::LoadFunc base_load_func = [&](const etl::Entry& etl_entry) { if (load_func) { load_func(etl_entry, target, flags); } else { - mdbx::slice k{db::to_slice(etl_entry.key)}; + mdbx::slice k = to_slice(etl_entry.key); if (etl_entry.value.empty()) { target.erase(k); } else { - mdbx::slice v{db::to_slice(etl_entry.value)}; + mdbx::slice v = to_slice(etl_entry.value); mdbx::error::success_or_throw(target.put(k, &v, flags)); } } @@ -55,4 +55,4 @@ class Collector : public etl::Collector { } }; -} // namespace silkworm::db::etl_mdbx \ No newline at end of file +} // namespace silkworm::sw_mdbx \ No newline at end of file diff --git a/silkworm/db/datastore/mdbx/mdbx.cpp b/silkworm/db/datastore/mdbx/mdbx.cpp index 600f193b04..11acb4761e 100644 --- a/silkworm/db/datastore/mdbx/mdbx.cpp +++ b/silkworm/db/datastore/mdbx/mdbx.cpp @@ -20,10 +20,10 @@ #include -namespace silkworm::db { +namespace silkworm::sw_mdbx { namespace detail { - std::string dump_mdbx_result(const db::CursorResult& result) { + std::string dump_mdbx_result(const CursorResult& result) { std::string dump{"done="}; dump.append(std::to_string(result.done)); dump.append(" bool(key)="); @@ -33,7 +33,7 @@ namespace detail { return dump; } - std::string slice_as_hex(const db::Slice& data) { + std::string slice_as_hex(const Slice& data) { return ::mdbx::to_hex(data).as_string(); } @@ -103,7 +103,7 @@ ::mdbx::env_managed open_env(const EnvConfig& config) { throw std::runtime_error("Path " + env_path.string() + " is not valid"); } - fs::path db_file{db::get_datafile_path(env_path)}; + fs::path db_file{get_datafile_path(env_path)}; const size_t db_file_size{fs::exists(db_file) ? fs::file_size(db_file) : 0}; if (!config.create && !db_file_size) { @@ -649,4 +649,4 @@ size_t cursor_erase(RWCursor& cursor, const ByteView set_key, const CursorMoveDi return ret; } -} // namespace silkworm::db +} // namespace silkworm::sw_mdbx diff --git a/silkworm/db/datastore/mdbx/mdbx.hpp b/silkworm/db/datastore/mdbx/mdbx.hpp index 71677b829d..f1cb432059 100644 --- a/silkworm/db/datastore/mdbx/mdbx.hpp +++ b/silkworm/db/datastore/mdbx/mdbx.hpp @@ -39,7 +39,7 @@ #include #include -namespace silkworm::db { +namespace silkworm::sw_mdbx { inline constexpr std::string_view kDbDataFileName{"mdbx.dat"}; @@ -71,8 +71,8 @@ namespace detail { void operator()(MDBX_cursor* ptr) const noexcept { mdbx_cursor_close(ptr); } }; - std::string dump_mdbx_result(const db::CursorResult& result); - std::string slice_as_hex(const db::Slice& data); + std::string dump_mdbx_result(const CursorResult& result); + std::string slice_as_hex(const Slice& data); } // namespace detail class ROTxn; @@ -597,4 +597,4 @@ inline ByteView from_slice(const Slice slice) { return {static_cast(slice.data()), slice.length()}; } -} // namespace silkworm::db +} // namespace silkworm::sw_mdbx diff --git a/silkworm/db/datastore/mdbx/mdbx_test.cpp b/silkworm/db/datastore/mdbx/mdbx_test.cpp index af3c88c307..6b0ff85da3 100644 --- a/silkworm/db/datastore/mdbx/mdbx_test.cpp +++ b/silkworm/db/datastore/mdbx/mdbx_test.cpp @@ -21,7 +21,7 @@ #include #include -namespace silkworm::db { +namespace silkworm::sw_mdbx { TEST_CASE("open_env") { // Empty dir @@ -67,4 +67,4 @@ TEST_CASE("open_env") { env.close(); } -} // namespace silkworm::db +} // namespace silkworm::sw_mdbx diff --git a/silkworm/db/datastore/mdbx/memory_mutation.cpp b/silkworm/db/datastore/mdbx/memory_mutation.cpp index a0413fa186..02a9ca2ee2 100644 --- a/silkworm/db/datastore/mdbx/memory_mutation.cpp +++ b/silkworm/db/datastore/mdbx/memory_mutation.cpp @@ -23,7 +23,7 @@ #include "memory_mutation_cursor.hpp" -namespace silkworm::db { +namespace silkworm::sw_mdbx { MemoryDatabase::MemoryDatabase(const std::filesystem::path& tmp_dir) { DataDirectory data_dir{tmp_dir}; @@ -36,7 +36,7 @@ MemoryDatabase::MemoryDatabase(const std::filesystem::path& tmp_dir) { .in_memory = true, .max_size = 512_Mebi, }; - memory_env_ = db::open_env(memory_config); + memory_env_ = open_env(memory_config); } MemoryDatabase::MemoryDatabase() : MemoryDatabase(TemporaryDirectory::get_unique_temporary_path()) {} @@ -47,7 +47,7 @@ ::mdbx::txn_managed MemoryDatabase::start_rw_txn() { MemoryOverlay::MemoryOverlay( const std::filesystem::path& tmp_dir, - silkworm::db::ROTxn* txn, + ROTxn* txn, std::function(const std::string& map_name)> get_map_config, std::string sequence_map_name) : memory_db_(tmp_dir), @@ -76,8 +76,8 @@ MemoryMutation::MemoryMutation(MemoryOverlay& overlay) overlay_(overlay) { // Initialize sequences const auto sequence_map_config = overlay.sequence_map_config(); - db::PooledCursor cursor{*overlay_.external_txn(), sequence_map_config}; - db::PooledCursor memory_cursor{managed_txn_, sequence_map_config}; + PooledCursor cursor{*overlay_.external_txn(), sequence_map_config}; + PooledCursor memory_cursor{managed_txn_, sequence_map_config}; for (auto result = cursor.to_first(false); result; result = cursor.to_next(false)) { memory_cursor.put(result.key, &result.value, MDBX_put_flags_t::MDBX_UPSERT); } @@ -119,7 +119,7 @@ bool MemoryMutation::is_dup_deleted(const std::string& table, const Slice& key, } bool MemoryMutation::has_map(const std::string& bucket_name) const { - return db::has_map(*overlay_.external_txn(), bucket_name.c_str()); + return sw_mdbx::has_map(*overlay_.external_txn(), bucket_name.c_str()); } void MemoryMutation::update_txn(ROTxn* txn) { @@ -174,7 +174,7 @@ bool MemoryMutation::clear_table(const std::string& table) { return managed_txn_.clear_map(table.c_str(), /*throw_if_absent=*/false); } -void MemoryMutation::flush(db::RWTxn& rw_txn) { +void MemoryMutation::flush(RWTxn& rw_txn) { reopen(); // Obliterate buckets that need to be deleted @@ -189,7 +189,7 @@ void MemoryMutation::flush(db::RWTxn& rw_txn) { SILK_WARN << "Unknown table " << table << " in memory mutation, ignored"; continue; } - const auto map_handle = db::open_map(rw_txn, *table_config); + const auto map_handle = open_map(rw_txn, *table_config); for (const auto& [key, _] : keys) { rw_txn->erase(map_handle, key); } @@ -202,7 +202,7 @@ void MemoryMutation::flush(db::RWTxn& rw_txn) { SILK_WARN << "Unknown table " << table << " in memory mutation, ignored"; continue; } - const auto map_handle = db::open_map(rw_txn, *table_config); + const auto map_handle = open_map(rw_txn, *table_config); for (const auto& [key, vals] : keys) { for (const auto& [val, _] : vals) { rw_txn->erase(map_handle, key, val); @@ -211,7 +211,7 @@ void MemoryMutation::flush(db::RWTxn& rw_txn) { } // Iterate over each touched bucket and apply changes accordingly - const auto tables = db::list_maps(managed_txn_); + const auto tables = list_maps(managed_txn_); for (const auto& table : tables) { const auto table_config = overlay_.map_config(table); if (!table_config) { @@ -250,4 +250,4 @@ std::unique_ptr MemoryMutation::make_cursor(const MapConfi return std::make_unique(*this, config); } -} // namespace silkworm::db +} // namespace silkworm::sw_mdbx diff --git a/silkworm/db/datastore/mdbx/memory_mutation.hpp b/silkworm/db/datastore/mdbx/memory_mutation.hpp index 7a1121bc58..7e941e5a93 100644 --- a/silkworm/db/datastore/mdbx/memory_mutation.hpp +++ b/silkworm/db/datastore/mdbx/memory_mutation.hpp @@ -23,7 +23,7 @@ #include "mdbx.hpp" -namespace silkworm::db { +namespace silkworm::sw_mdbx { class MemoryDatabase { public: @@ -50,7 +50,7 @@ class MemoryOverlay { MemoryOverlay(MemoryOverlay&& other) noexcept = default; MemoryOverlay& operator=(MemoryOverlay&&) noexcept = default; - db::ROTxn* external_txn() const { return txn_; } + ROTxn* external_txn() const { return txn_; } void update_txn(ROTxn* txn); ::mdbx::txn_managed start_rw_txn(); @@ -81,7 +81,7 @@ class MemoryMutation : public RWTxnManaged { bool is_dup_deleted(const std::string& table, const Slice& key, const Slice& value) const; bool has_map(const std::string& bucket_name) const; - db::ROTxn* external_txn() const { return overlay_.external_txn(); } + ROTxn* external_txn() const { return overlay_.external_txn(); } void update_txn(ROTxn* txn); @@ -96,7 +96,7 @@ class MemoryMutation : public RWTxnManaged { bool clear_table(const std::string& table); - void flush(db::RWTxn& rw_txn); + void flush(RWTxn& rw_txn); void rollback(); void reopen(); @@ -109,4 +109,4 @@ class MemoryMutation : public RWTxnManaged { std::map cleared_tables_; }; -} // namespace silkworm::db +} // namespace silkworm::sw_mdbx diff --git a/silkworm/db/datastore/mdbx/memory_mutation_cursor.cpp b/silkworm/db/datastore/mdbx/memory_mutation_cursor.cpp index 3139cda883..841fe2e7fd 100644 --- a/silkworm/db/datastore/mdbx/memory_mutation_cursor.cpp +++ b/silkworm/db/datastore/mdbx/memory_mutation_cursor.cpp @@ -18,7 +18,7 @@ #include -namespace silkworm::db { +namespace silkworm::sw_mdbx { MemoryMutationCursor::MemoryMutationCursor(MemoryMutation& memory_mutation, const MapConfig& config) : memory_mutation_(memory_mutation), @@ -835,4 +835,4 @@ void MemoryMutationCursor::throw_error_notfound() { mdbx::error::throw_exception(MDBX_error_t::MDBX_NOTFOUND); } -} // namespace silkworm::db +} // namespace silkworm::sw_mdbx diff --git a/silkworm/db/datastore/mdbx/memory_mutation_cursor.hpp b/silkworm/db/datastore/mdbx/memory_mutation_cursor.hpp index 259755d310..3d29b52507 100644 --- a/silkworm/db/datastore/mdbx/memory_mutation_cursor.hpp +++ b/silkworm/db/datastore/mdbx/memory_mutation_cursor.hpp @@ -21,7 +21,7 @@ #include "mdbx.hpp" #include "memory_mutation.hpp" -namespace silkworm::db { +namespace silkworm::sw_mdbx { class MemoryMutationCursor : public RWCursorDupSort { public: @@ -118,4 +118,4 @@ class MemoryMutationCursor : public RWCursorDupSort { bool is_previous_from_db_{false}; }; -} // namespace silkworm::db +} // namespace silkworm::sw_mdbx diff --git a/silkworm/db/datastore/mdbx/memory_mutation_test.cpp b/silkworm/db/datastore/mdbx/memory_mutation_test.cpp index 627e60f9c5..6aafdcfb9e 100644 --- a/silkworm/db/datastore/mdbx/memory_mutation_test.cpp +++ b/silkworm/db/datastore/mdbx/memory_mutation_test.cpp @@ -20,7 +20,7 @@ #include -namespace silkworm::db { +namespace silkworm::sw_mdbx { static const MapConfig kTestMap{"TestTable"}; static const MapConfig kTestMultiMap{"TestMultiTable", mdbx::key_mode::usual, mdbx::value_mode::multi}; @@ -53,12 +53,12 @@ TEST_CASE("MemoryMutation", "[silkworm][node][db][memory_mutation]") { const TemporaryDirectory tmp_dir; DataDirectory data_dir{tmp_dir.path() / "main_db"}; data_dir.deploy(); - db::EnvConfig main_db_config{ + EnvConfig main_db_config{ .path = data_dir.chaindata().path().string(), .create = true, .in_memory = true, }; - auto main_env{db::open_env(main_db_config)}; + auto main_env{open_env(main_db_config)}; RWTxnManaged main_rw_txn{main_env}; MemoryOverlay overlay{ @@ -67,7 +67,7 @@ TEST_CASE("MemoryMutation", "[silkworm][node][db][memory_mutation]") { [](const std::string& map_name) { if (map_name == kTestMap.name) return kTestMap; if (map_name == kTestMultiMap.name) return kTestMultiMap; - return db::MapConfig{map_name.c_str()}; + return MapConfig{map_name.c_str()}; }, "Sequence", }; @@ -289,4 +289,4 @@ TEST_CASE("MemoryMutation", "[silkworm][node][db][memory_mutation]") { } } -} // namespace silkworm::db +} // namespace silkworm::sw_mdbx diff --git a/silkworm/db/datastore/segment_collation.hpp b/silkworm/db/datastore/segment_collation.hpp index 2c42bbd227..5618779b0b 100644 --- a/silkworm/db/datastore/segment_collation.hpp +++ b/silkworm/db/datastore/segment_collation.hpp @@ -24,7 +24,7 @@ #include "mdbx/mdbx.hpp" #include "snapshots/segment/segment_writer.hpp" -namespace silkworm::db { +namespace silkworm::datastore { struct SegmentCollationCommand : public DataMigrationCommand { BlockNumRange range; @@ -46,10 +46,10 @@ struct SegmentCollation { virtual ~SegmentCollation() = default; //! Copies data for a block range from db to the snapshot file. - virtual void copy(ROTxn& txn, const SegmentCollationCommand& command, snapshots::segment::SegmentFileWriter& file_writer) const = 0; + virtual void copy(sw_mdbx::ROTxn& txn, const SegmentCollationCommand& command, snapshots::segment::SegmentFileWriter& file_writer) const = 0; //! Cleans up data for a block range from db after it was copied to the snapshot file. - virtual void prune(RWTxn& txn, BlockNumRange range) const = 0; + virtual void prune(sw_mdbx::RWTxn& txn, BlockNumRange range) const = 0; }; -} // namespace silkworm::db +} // namespace silkworm::datastore diff --git a/silkworm/db/datastore/snapshot_merger.cpp b/silkworm/db/datastore/snapshot_merger.cpp index dd32b558f0..995193b7c7 100644 --- a/silkworm/db/datastore/snapshot_merger.cpp +++ b/silkworm/db/datastore/snapshot_merger.cpp @@ -29,7 +29,7 @@ #include "snapshots/segment/segment_writer.hpp" #include "snapshots/snapshot_bundle.hpp" -namespace silkworm::db { +namespace silkworm::datastore { using namespace silkworm::snapshots; @@ -144,4 +144,4 @@ Task SnapshotMerger::cleanup() { co_return; } -} // namespace silkworm::db +} // namespace silkworm::datastore diff --git a/silkworm/db/datastore/snapshot_merger.hpp b/silkworm/db/datastore/snapshot_merger.hpp index 2cb586b305..062a24c932 100644 --- a/silkworm/db/datastore/snapshot_merger.hpp +++ b/silkworm/db/datastore/snapshot_merger.hpp @@ -26,7 +26,7 @@ #include "snapshots/snapshot_repository.hpp" #include "snapshots/snapshot_size.hpp" -namespace silkworm::db { +namespace silkworm::datastore { class SnapshotMerger : public DataMigration { public: @@ -54,4 +54,4 @@ class SnapshotMerger : public DataMigration { boost::signals2::signal on_snapshot_merged_signal_; }; -} // namespace silkworm::db +} // namespace silkworm::datastore diff --git a/silkworm/db/datastore/snapshots/index_builder.hpp b/silkworm/db/datastore/snapshots/index_builder.hpp index 9fe23ee4c2..a9fba8ffb2 100644 --- a/silkworm/db/datastore/snapshots/index_builder.hpp +++ b/silkworm/db/datastore/snapshots/index_builder.hpp @@ -42,7 +42,7 @@ struct IndexDescriptor { uint64_t base_data_id{}; bool double_enum_index{true}; bool less_false_positives{}; - size_t etl_buffer_size{db::etl::kOptimalBufferSize}; + size_t etl_buffer_size{etl::kOptimalBufferSize}; }; struct IndexInputDataQuery { diff --git a/silkworm/db/datastore/snapshots/rec_split/rec_split_seq.hpp b/silkworm/db/datastore/snapshots/rec_split/rec_split_seq.hpp index deeada034b..f5dcea1cee 100644 --- a/silkworm/db/datastore/snapshots/rec_split/rec_split_seq.hpp +++ b/silkworm/db/datastore/snapshots/rec_split/rec_split_seq.hpp @@ -68,8 +68,8 @@ struct RecSplit::SequentialBuildingStrategy : public BuildingStrategy protected: void setup(const RecSplitSettings& settings, size_t bucket_count) override { - offset_collector_ = std::make_unique(etl_optimal_size_); - bucket_collector_ = std::make_unique(etl_optimal_size_); + offset_collector_ = std::make_unique(etl_optimal_size_); + bucket_collector_ = std::make_unique(etl_optimal_size_); bucket_size_accumulator_.reserve(bucket_count + 1); bucket_position_accumulator_.reserve(bucket_count + 1); @@ -124,7 +124,7 @@ struct RecSplit::SequentialBuildingStrategy : public BuildingStrategy }; try { // Not passing any cursor is a valid use-case for ETL when DB modification is not expected - bucket_collector_->load([&](const db::etl::Entry& entry) { + bucket_collector_->load([&](const etl::Entry& entry) { // k is the big-endian encoding of the bucket number and the v is the key that is assigned into that bucket const uint64_t bucket_id = endian::load_big_u64(entry.key.data()); SILK_TRACE << "[index] processing bucket_id=" << bucket_id; @@ -163,7 +163,7 @@ struct RecSplit::SequentialBuildingStrategy : public BuildingStrategy void build_enum_index(std::unique_ptr& ef_offsets) override { // Build Elias-Fano index for offsets (if any) ef_offsets = std::make_unique(keys_added_, max_offset_); - offset_collector_->load([&](const db::etl::Entry& entry) { + offset_collector_->load([&](const etl::Entry& entry) { const uint64_t offset = endian::load_big_u64(entry.key.data()); ef_offsets->add_offset(offset); }); @@ -240,7 +240,7 @@ struct RecSplit::SequentialBuildingStrategy : public BuildingStrategy private: // Optimal size for offset and bucket ETL collectors - size_t etl_optimal_size_{db::etl::kOptimalBufferSize}; + size_t etl_optimal_size_{etl::kOptimalBufferSize}; //! Flag indicating if two-level index "recsplit -> enum" + "enum -> offset" is required bool double_enum_index_{false}; @@ -261,10 +261,10 @@ struct RecSplit::SequentialBuildingStrategy : public BuildingStrategy std::vector current_bucket_offsets_; //! The ETL collector sorting keys by offset - std::unique_ptr offset_collector_{}; + std::unique_ptr offset_collector_{}; //! The ETL collector sorting keys by bucket - std::unique_ptr bucket_collector_{}; + std::unique_ptr bucket_collector_{}; //! Accumulator for size of every bucket std::vector bucket_size_accumulator_; @@ -282,7 +282,7 @@ struct RecSplit::SequentialBuildingStrategy : public BuildingStrategy GolombRiceBuilder gr_builder_; }; -inline auto seq_build_strategy(size_t etl_buffer_size = db::etl::kOptimalBufferSize) { +inline auto seq_build_strategy(size_t etl_buffer_size = etl::kOptimalBufferSize) { return std::make_unique(etl_buffer_size); } diff --git a/silkworm/db/datastore/snapshots/rec_split/rec_split_seq_test.cpp b/silkworm/db/datastore/snapshots/rec_split/rec_split_seq_test.cpp index be5b15bce3..fae3927a67 100644 --- a/silkworm/db/datastore/snapshots/rec_split/rec_split_seq_test.cpp +++ b/silkworm/db/datastore/snapshots/rec_split/rec_split_seq_test.cpp @@ -123,7 +123,7 @@ template <> const std::array RecSplit4::kMemo = RecSplit4::fill_golomb_rice(); using RecSplit4 = RecSplit; -auto seq_build_strategy_4() { return std::make_unique(db::etl::kOptimalBufferSize); } +auto seq_build_strategy_4() { return std::make_unique(etl::kOptimalBufferSize); } TEST_CASE("RecSplit4: keys=1000 buckets=128", "[silkworm][snapshots][recsplit]") { TemporaryFile index_file; diff --git a/silkworm/db/datastore/snapshots/segment/seg/CMakeLists.txt b/silkworm/db/datastore/snapshots/segment/seg/CMakeLists.txt index ccd3ea654a..c803fe2e7e 100644 --- a/silkworm/db/datastore/snapshots/segment/seg/CMakeLists.txt +++ b/silkworm/db/datastore/snapshots/segment/seg/CMakeLists.txt @@ -22,5 +22,5 @@ find_package(Boost REQUIRED COMPONENTS headers iostreams) silkworm_library( silkworm_snapshots_seg PUBLIC silkworm_core silkworm_infra - PRIVATE Boost::headers Boost::iostreams sais_lite silkworm_db_etl + PRIVATE Boost::headers Boost::iostreams sais_lite silkworm_etl ) diff --git a/silkworm/db/datastore/snapshots/segment/seg/compressor/pattern_aggregator.cpp b/silkworm/db/datastore/snapshots/segment/seg/compressor/pattern_aggregator.cpp index 6e15af863d..a10bd07634 100644 --- a/silkworm/db/datastore/snapshots/segment/seg/compressor/pattern_aggregator.cpp +++ b/silkworm/db/datastore/snapshots/segment/seg/compressor/pattern_aggregator.cpp @@ -48,7 +48,7 @@ class PatternAggregatorImpl { public: explicit PatternAggregatorImpl(const filesystem::path& etl_work_path) : etl_work_path_(etl_work_path), - collector_(make_unique(etl_work_path, db::etl::kOptimalBufferSize / 4)) {} + collector_(make_unique(etl_work_path, etl::kOptimalBufferSize / 4)) {} void collect_pattern(Pattern pattern) { collector_->collect(std::move(pattern.data), big_endian_encode(pattern.score)); @@ -67,20 +67,20 @@ class PatternAggregatorImpl { static constexpr size_t kMaxPatternsSoftLimit = 1'000'000; // destination = SELECT data, sum(score) FROM source GROUP BY data - static void sum_score_group_by_pattern(db::etl::Collector& source, db::etl::Collector& destination); + static void sum_score_group_by_pattern(etl::Collector& source, etl::Collector& destination); // SELECT * FROM source ORDER BY score DESC LIMIT kMaxPatterns - static vector order_by_score_and_limit(db::etl::Collector& source); + static vector order_by_score_and_limit(etl::Collector& source); filesystem::path etl_work_path_; - unique_ptr collector_; + unique_ptr collector_; }; -void PatternAggregatorImpl::sum_score_group_by_pattern(db::etl::Collector& source, db::etl::Collector& destination) { +void PatternAggregatorImpl::sum_score_group_by_pattern(etl::Collector& source, etl::Collector& destination) { optional pending_pattern_data; uint64_t pending_pattern_score{}; - source.load([&](const db::etl::Entry& entry) { + source.load([&](const etl::Entry& entry) { auto& pattern_data = entry.key; uint64_t pattern_score = big_endian_decode(entry.value); @@ -99,11 +99,11 @@ void PatternAggregatorImpl::sum_score_group_by_pattern(db::etl::Collector& sourc } } -vector PatternAggregatorImpl::order_by_score_and_limit(db::etl::Collector& source) { +vector PatternAggregatorImpl::order_by_score_and_limit(etl::Collector& source) { vector patterns; std::greater<> comparator; - source.load([&](const db::etl::Entry& entry) { + source.load([&](const etl::Entry& entry) { Pattern pattern{ .data = Bytes{entry.key}, .score = big_endian_decode(entry.value), @@ -128,7 +128,7 @@ vector PatternAggregatorImpl::order_by_score_and_limit(db::etl::Collect } vector PatternAggregatorImpl::aggregate() { - db::etl::Collector collector{etl_work_path_}; + etl::Collector collector{etl_work_path_}; sum_score_group_by_pattern(*collector_, collector); // the original collector_ is not needed anymore, free it diff --git a/silkworm/db/datastore/stage_scheduler.hpp b/silkworm/db/datastore/stage_scheduler.hpp index 6a34e52777..c1891ce37f 100644 --- a/silkworm/db/datastore/stage_scheduler.hpp +++ b/silkworm/db/datastore/stage_scheduler.hpp @@ -22,13 +22,13 @@ #include "mdbx/mdbx.hpp" -namespace silkworm::stagedsync { +namespace silkworm::datastore { struct StageScheduler { virtual ~StageScheduler() = default; //! Schedule a callback to run inside the stage loop. - virtual Task schedule(std::function callback) = 0; + virtual Task schedule(std::function callback) = 0; }; -} // namespace silkworm::stagedsync +} // namespace silkworm::datastore diff --git a/silkworm/db/db_utils.cpp b/silkworm/db/db_utils.cpp index 1b03b2e6c1..b1de677fad 100644 --- a/silkworm/db/db_utils.cpp +++ b/silkworm/db/db_utils.cpp @@ -20,11 +20,11 @@ #include #include -namespace silkworm { +namespace silkworm::db { // Return (block-num, hash) of the header with the biggest total difficulty skipping bad headers // see Erigon's HeadersUnwind method for the implementation -std::tuple header_with_biggest_td(db::ROTxn& txn, const std::set* bad_headers) { +std::tuple header_with_biggest_td(sw_mdbx::ROTxn& txn, const std::set* bad_headers) { BlockNum max_block_num = 0; Hash max_hash; intx::uint256 max_td = 0; @@ -53,9 +53,9 @@ std::tuple header_with_biggest_td(db::ROTxn& txn, const std::set // TODO: check if we really need to parse all the table }; - db::cursor_for_each(*td_cursor, find_max, db::CursorMoveDirection::kReverse); + sw_mdbx::cursor_for_each(*td_cursor, find_max, sw_mdbx::CursorMoveDirection::kReverse); return {max_block_num, max_hash}; } -} // namespace silkworm +} // namespace silkworm::db diff --git a/silkworm/db/db_utils.hpp b/silkworm/db/db_utils.hpp index e40dc1ddb1..b1b07b5725 100644 --- a/silkworm/db/db_utils.hpp +++ b/silkworm/db/db_utils.hpp @@ -25,9 +25,9 @@ #include #include -namespace silkworm { +namespace silkworm::db { //! \brief Return (block-num, hash) of the header with the biggest total difficulty skipping bad headers -std::tuple header_with_biggest_td(db::ROTxn& txn, const std::set* bad_headers = nullptr); +std::tuple header_with_biggest_td(sw_mdbx::ROTxn& txn, const std::set* bad_headers = nullptr); -} // namespace silkworm +} // namespace silkworm::db diff --git a/silkworm/db/db_utils_test.cpp b/silkworm/db/db_utils_test.cpp index d844e19d25..fdac7b5622 100644 --- a/silkworm/db/db_utils_test.cpp +++ b/silkworm/db/db_utils_test.cpp @@ -25,6 +25,8 @@ namespace silkworm::db { +using namespace sw_mdbx; + TEST_CASE("db access layer addendum") { TemporaryDirectory tmp_dir; diff --git a/silkworm/db/etl_collector_test.cpp b/silkworm/db/etl_collector_test.cpp index 6bf022861a..373ca9d5f6 100644 --- a/silkworm/db/etl_collector_test.cpp +++ b/silkworm/db/etl_collector_test.cpp @@ -28,9 +28,10 @@ #include #include -namespace silkworm::db::etl { +namespace silkworm::sw_mdbx { namespace fs = std::filesystem; +using silkworm::etl::Entry; static std::vector generate_entry_set(size_t size) { std::vector pairs; @@ -57,7 +58,7 @@ static std::vector generate_entry_set(size_t size) { return pairs; } -void run_collector_test(const etl_mdbx::LoadFunc& load_func, bool do_copy = true) { +void run_collector_test(const sw_mdbx::LoadFunc& load_func, bool do_copy = true) { db::test_util::TempChainData context; // Generate Test Entries @@ -68,7 +69,7 @@ void run_collector_test(const etl_mdbx::LoadFunc& load_func, bool do_copy = true } // expect 10 files - etl_mdbx::Collector collector{context.dir().temp().path(), generated_size / 10}; + sw_mdbx::Collector collector{context.dir().temp().path(), generated_size / 10}; // Collection for (auto&& entry : set) { @@ -91,7 +92,7 @@ void run_collector_test(const etl_mdbx::LoadFunc& load_func, bool do_copy = true } }); - db::PooledCursor to{context.rw_txn(), db::table::kHeaderNumbers}; + sw_mdbx::PooledCursor to{context.rw_txn(), db::table::kHeaderNumbers}; collector.load(to, load_func); // Check whether temporary files were cleaned CHECK(std::distance(fs::directory_iterator{context.dir().temp().path()}, fs::directory_iterator{}) == 0); @@ -110,8 +111,8 @@ TEST_CASE("collect_and_load") { run_collector_test([](const Entry& entry, auto& table, MDBX_put_flags_t) { Bytes key{entry.key}; key.at(0) = 1; - table.upsert(db::to_slice(key), db::to_slice(entry.value)); + table.upsert(sw_mdbx::to_slice(key), sw_mdbx::to_slice(entry.value)); }); } -} // namespace silkworm::db::etl +} // namespace silkworm::sw_mdbx diff --git a/silkworm/db/etl_in_memory_collector_test.cpp b/silkworm/db/etl_in_memory_collector_test.cpp index a8ec6d63fa..3aa93851d4 100644 --- a/silkworm/db/etl_in_memory_collector_test.cpp +++ b/silkworm/db/etl_in_memory_collector_test.cpp @@ -28,7 +28,7 @@ #include "util.hpp" -namespace silkworm::db::etl_mdbx { +namespace silkworm::sw_mdbx { // Function pointer to process Load on before Load data into tables using KVLoadFunc = std::function generate_entry_set(size_t size) { template void run_collector_test(const KVLoadFunc& load_func, bool do_copy = true) { - test_util::TempChainData context; + db::test_util::TempChainData context; // Generate Test Entries auto set{generate_entry_set(1000)}; // 1000 entries in total @@ -130,12 +130,12 @@ void run_collector_test(const KVLoadFunc& load_func, bool do_copy = true) { }); // Load data - PooledCursor to{context.rw_txn(), table::kHeaderNumbers}; + PooledCursor to{context.rw_txn(), db::table::kHeaderNumbers}; collector.load(to, load_func); if (load_func) { size_t found_items{0}; - PooledCursor from{context.rw_txn(), table::kHeaderNumbers}; + PooledCursor from{context.rw_txn(), db::table::kHeaderNumbers}; auto data = from.to_first(); while (data) { auto key = from_slice(data.key); @@ -185,4 +185,4 @@ TEST_CASE("collect_and_load_in_memory_vector") { }); } -} // namespace silkworm::db::etl_mdbx \ No newline at end of file +} // namespace silkworm::sw_mdbx \ No newline at end of file diff --git a/silkworm/db/freezer.cpp b/silkworm/db/freezer.cpp index eb8b4ffb8b..16eab2db66 100644 --- a/silkworm/db/freezer.cpp +++ b/silkworm/db/freezer.cpp @@ -39,6 +39,7 @@ namespace silkworm::db { +using namespace silkworm::datastore; using namespace silkworm::snapshots; struct FreezerResult : public DataMigrationResult { diff --git a/silkworm/db/freezer.hpp b/silkworm/db/freezer.hpp index 3276f6700c..1cf26312bb 100644 --- a/silkworm/db/freezer.hpp +++ b/silkworm/db/freezer.hpp @@ -23,12 +23,15 @@ namespace silkworm::db { -class Freezer : public DataMigration { +class Freezer : public datastore::DataMigration { public: + using DataMigrationCommand = datastore::DataMigrationCommand; + using DataMigrationResult = datastore::DataMigrationResult; + Freezer( - db::ROAccess db_access, + sw_mdbx::ROAccess db_access, snapshots::SnapshotRepository& snapshots, - stagedsync::StageScheduler& stage_scheduler, + datastore::StageScheduler& stage_scheduler, std::filesystem::path tmp_dir_path, bool keep_blocks) : db_access_(std::move(db_access)), @@ -47,11 +50,11 @@ class Freezer : public DataMigration { void commit(std::shared_ptr result) override; Task cleanup() override; BlockNumRange cleanup_range(); - void prune_collations(RWTxn& db_tx, BlockNumRange range) const; + void prune_collations(sw_mdbx::RWTxn& db_tx, BlockNumRange range) const; - db::ROAccess db_access_; + sw_mdbx::ROAccess db_access_; snapshots::SnapshotRepository& snapshots_; - stagedsync::StageScheduler& stage_scheduler_; + datastore::StageScheduler& stage_scheduler_; std::filesystem::path tmp_dir_path_; bool keep_blocks_; }; diff --git a/silkworm/db/genesis.cpp b/silkworm/db/genesis.cpp index e146031c5f..887dee21b3 100644 --- a/silkworm/db/genesis.cpp +++ b/silkworm/db/genesis.cpp @@ -27,6 +27,8 @@ namespace silkworm::db { +using sw_mdbx::to_slice; + std::pair> validate_genesis_json(const nlohmann::json& genesis_json) { std::pair> ret{true, {}}; if (genesis_json.is_discarded()) { diff --git a/silkworm/db/genesis.hpp b/silkworm/db/genesis.hpp index ed61cd6909..34ea7e419c 100644 --- a/silkworm/db/genesis.hpp +++ b/silkworm/db/genesis.hpp @@ -21,6 +21,7 @@ #include namespace silkworm::db { + //! \brief Validates provided genesis json payload //! \param [in] genesis_json : the payload to validate //! \returns A pair of bool and a vector of string errors (if any) diff --git a/silkworm/db/genesis_test.cpp b/silkworm/db/genesis_test.cpp index 701acca813..dea59b4451 100644 --- a/silkworm/db/genesis_test.cpp +++ b/silkworm/db/genesis_test.cpp @@ -118,7 +118,7 @@ TEST_CASE("Database genesis initialization") { SECTION("Without genesis block") { // Nothing should happen update_chain_config(txn, kMainnetConfig); - PooledCursor config(txn, table::kConfig); + sw_mdbx::PooledCursor config(txn, table::kConfig); REQUIRE(config.empty()); } diff --git a/silkworm/db/kv/api/local_cursor.cpp b/silkworm/db/kv/api/local_cursor.cpp index 877800ffae..57e6478c6b 100644 --- a/silkworm/db/kv/api/local_cursor.cpp +++ b/silkworm/db/kv/api/local_cursor.cpp @@ -23,7 +23,8 @@ namespace silkworm::db::kv::api { -using detail::slice_as_hex; +using namespace sw_mdbx; +using sw_mdbx::detail::slice_as_hex; Task LocalCursor::open_cursor(const std::string& table_name, bool is_dup_sorted) { const auto start_time = clock_time::now(); diff --git a/silkworm/db/kv/api/local_cursor.hpp b/silkworm/db/kv/api/local_cursor.hpp index f45b9a04d1..a6fc14529e 100644 --- a/silkworm/db/kv/api/local_cursor.hpp +++ b/silkworm/db/kv/api/local_cursor.hpp @@ -61,7 +61,7 @@ class LocalCursor : public CursorDupSort { private: uint32_t cursor_id_; - PooledCursor db_cursor_; + sw_mdbx::PooledCursor db_cursor_; mdbx::txn& txn_; }; diff --git a/silkworm/db/kv/api/local_cursor_test.cpp b/silkworm/db/kv/api/local_cursor_test.cpp index 299e099b5b..ecf7e49ade 100644 --- a/silkworm/db/kv/api/local_cursor_test.cpp +++ b/silkworm/db/kv/api/local_cursor_test.cpp @@ -30,6 +30,7 @@ namespace silkworm::db::kv::api { using namespace silkworm::test_util; using silkworm::test_util::ContextTestBase; +using sw_mdbx::ROTxnManaged; using test_util::TestDatabaseContext; struct LocalCursorTest : public ContextTestBase { @@ -37,7 +38,7 @@ struct LocalCursorTest : public ContextTestBase { TestDatabaseContext database{tmp_dir}; static inline uint32_t last_cursor_id{0}; - db::ROAccess chaindata() const { return database.chaindata(); } + sw_mdbx::ROAccess chaindata() const { return database.chaindata(); } }; // In all following tests we need to create the MDBX transaction using the io_context scheduler thread, so we simply diff --git a/silkworm/db/kv/api/local_transaction.hpp b/silkworm/db/kv/api/local_transaction.hpp index 87a0216b93..b05685dba3 100644 --- a/silkworm/db/kv/api/local_transaction.hpp +++ b/silkworm/db/kv/api/local_transaction.hpp @@ -90,7 +90,7 @@ class LocalTransaction : public BaseTransaction { DataStoreRef data_store_; uint32_t last_cursor_id_{0}; - ROTxnManaged txn_; + sw_mdbx::ROTxnManaged txn_; uint64_t tx_id_{++next_tx_id_}; }; diff --git a/silkworm/db/kv/grpc/server/kv_calls.hpp b/silkworm/db/kv/grpc/server/kv_calls.hpp index 930c5e2115..1a6d418755 100644 --- a/silkworm/db/kv/grpc/server/kv_calls.hpp +++ b/silkworm/db/kv/grpc/server/kv_calls.hpp @@ -42,6 +42,8 @@ namespace silkworm::db::kv::grpc::server { +using namespace sw_mdbx; + api::Version higher_version_ignoring_patch(api::Version lhs, api::Version rhs); //! Current DB schema version. diff --git a/silkworm/db/kv/grpc/server/kv_calls_test.cpp b/silkworm/db/kv/grpc/server/kv_calls_test.cpp index e9f44c649d..f22ca67499 100644 --- a/silkworm/db/kv/grpc/server/kv_calls_test.cpp +++ b/silkworm/db/kv/grpc/server/kv_calls_test.cpp @@ -62,28 +62,30 @@ TEST_CASE("higher_version_ignoring_patch", "[silkworm][rpc][kv_calls]") { } } -static const silkworm::db::MapConfig kTestMap{"TestTable"}; +static const sw_mdbx::MapConfig kTestMap{"TestTable"}; TEST_CASE("dump_mdbx_result", "[silkworm][rpc][kv_calls]") { + using namespace sw_mdbx; + TemporaryDirectory tmp_dir; DataDirectory data_dir{tmp_dir.path()}; REQUIRE_NOTHROW(data_dir.deploy()); - db::EnvConfig db_config; + EnvConfig db_config; db_config.path = data_dir.chaindata().path().string(); db_config.create = true; db_config.in_memory = true; - auto database_env = db::open_env(db_config); + auto database_env = open_env(db_config); auto rw_txn{database_env.start_write()}; - db::open_map(rw_txn, kTestMap); - db::PooledCursor rw_cursor{rw_txn, kTestMap}; + open_map(rw_txn, kTestMap); + PooledCursor rw_cursor{rw_txn, kTestMap}; rw_cursor.upsert(mdbx::slice{"AA"}, mdbx::slice{"00"}); rw_cursor.upsert(mdbx::slice{"BB"}, mdbx::slice{"11"}); rw_txn.commit(); auto ro_txn = database_env.start_read(); - db::PooledCursor cursor{ro_txn, kTestMap}; - db::CursorResult result = cursor.to_first(/*throw_notfound=*/false); - const auto result_dump = db::detail::dump_mdbx_result(result); + PooledCursor cursor{ro_txn, kTestMap}; + CursorResult result = cursor.to_first(/*throw_notfound=*/false); + const auto result_dump = sw_mdbx::detail::dump_mdbx_result(result); CHECK(result_dump.find(std::to_string(result.done)) != std::string::npos); CHECK(result_dump.find(std::to_string(static_cast(result.key))) != std::string::npos); CHECK(result_dump.find(std::to_string(static_cast(result.value))) != std::string::npos); diff --git a/silkworm/db/kv/grpc/server/kv_server.hpp b/silkworm/db/kv/grpc/server/kv_server.hpp index 11fc673e28..b5e2d51de9 100644 --- a/silkworm/db/kv/grpc/server/kv_server.hpp +++ b/silkworm/db/kv/grpc/server/kv_server.hpp @@ -32,7 +32,7 @@ class KvServer : public virtual rpc::Server { public: KvServer( const rpc::ServerSettings& settings, - ROAccess chaindata, + sw_mdbx::ROAccess chaindata, StateChangeCollection* state_change_source); KvServer(const KvServer&) = delete; @@ -50,7 +50,7 @@ class KvServer : public virtual rpc::Server { remote::KV::AsyncService kv_async_service_; //! The chain database environment - ROAccess chaindata_; + sw_mdbx::ROAccess chaindata_; //! The collector of state changes acting as source of state change notifications StateChangeCollection* state_change_source_; diff --git a/silkworm/db/kv/grpc/server/kv_server_test.cpp b/silkworm/db/kv/grpc/server/kv_server_test.cpp index 905a518bd8..1d7ec0fff5 100644 --- a/silkworm/db/kv/grpc/server/kv_server_test.cpp +++ b/silkworm/db/kv/grpc/server/kv_server_test.cpp @@ -179,8 +179,8 @@ class ThreadedKvClient { // TODO(canepat): better copy grpc_pick_unused_port_or_die to generate unused port const std::string kTestAddressUri{"localhost:12345"}; -const silkworm::db::MapConfig kTestMap{"TestTable"}; -const silkworm::db::MapConfig kTestMultiMap{"TestMultiTable", mdbx::key_mode::usual, mdbx::value_mode::multi}; +const silkworm::sw_mdbx::MapConfig kTestMap{"TestTable"}; +const silkworm::sw_mdbx::MapConfig kTestMultiMap{"TestMultiTable", mdbx::key_mode::usual, mdbx::value_mode::multi}; using namespace silkworm; @@ -218,27 +218,27 @@ struct KvEnd2EndTest { DataDirectory data_dir{tmp_dir.path()}; REQUIRE_NOTHROW(data_dir.deploy()); - db_config = std::make_unique(); + db_config = std::make_unique(); db_config->path = data_dir.chaindata().path().string(); db_config->create = true; db_config->in_memory = true; - database_env = db::open_env(*db_config); + database_env = sw_mdbx::open_env(*db_config); auto rw_txn{database_env.start_write()}; - db::open_map(rw_txn, kTestMap); - db::open_map(rw_txn, kTestMultiMap); + sw_mdbx::open_map(rw_txn, kTestMap); + sw_mdbx::open_map(rw_txn, kTestMultiMap); rw_txn.commit(); state_change_collection = std::make_unique(); - server = std::make_unique(srv_config, db::ROAccess{database_env}, state_change_collection.get()); + server = std::make_unique(srv_config, sw_mdbx::ROAccess{database_env}, state_change_collection.get()); server->build_and_start(); } void fill_tables() { auto rw_txn = database_env.start_write(); - db::PooledCursor rw_cursor1{rw_txn, kTestMap}; + sw_mdbx::PooledCursor rw_cursor1{rw_txn, kTestMap}; rw_cursor1.upsert(mdbx::slice{"AA"}, mdbx::slice{"00"}); rw_cursor1.upsert(mdbx::slice{"BB"}, mdbx::slice{"11"}); - db::PooledCursor rw_cursor2{rw_txn, kTestMultiMap}; + sw_mdbx::PooledCursor rw_cursor2{rw_txn, kTestMultiMap}; rw_cursor2.upsert(mdbx::slice{"AA"}, mdbx::slice{"00"}); rw_cursor2.upsert(mdbx::slice{"AA"}, mdbx::slice{"11"}); rw_cursor2.upsert(mdbx::slice{"AA"}, mdbx::slice{"22"}); @@ -248,9 +248,9 @@ struct KvEnd2EndTest { void alter_tables() { auto rw_txn = database_env.start_write(); - db::PooledCursor rw_cursor1{rw_txn, kTestMap}; + sw_mdbx::PooledCursor rw_cursor1{rw_txn, kTestMap}; rw_cursor1.upsert(mdbx::slice{"CC"}, mdbx::slice{"22"}); - db::PooledCursor rw_cursor2{rw_txn, kTestMultiMap}; + sw_mdbx::PooledCursor rw_cursor2{rw_txn, kTestMultiMap}; rw_cursor2.upsert(mdbx::slice{"AA"}, mdbx::slice{"33"}); rw_cursor2.upsert(mdbx::slice{"BB"}, mdbx::slice{"33"}); rw_txn.commit(); @@ -266,7 +266,7 @@ struct KvEnd2EndTest { std::unique_ptr kv_client; rpc::ServerSettings srv_config; TemporaryDirectory tmp_dir; - std::unique_ptr db_config; + std::unique_ptr db_config; mdbx::env_managed database_env; std::unique_ptr state_change_collection; std::unique_ptr server; @@ -285,10 +285,10 @@ TEST_CASE("KvServer", "[silkworm][node][rpc]") { TemporaryDirectory tmp_dir; DataDirectory data_dir{tmp_dir.path()}; REQUIRE_NOTHROW(data_dir.deploy()); - db::EnvConfig db_config{data_dir.chaindata().path().string()}; + sw_mdbx::EnvConfig db_config{data_dir.chaindata().path().string()}; db_config.create = true; db_config.in_memory = true; - auto chaindata_env = db::open_env(db_config); + auto chaindata_env = sw_mdbx::open_env(db_config); ROAccess chaindata{chaindata_env}; auto state_change_source{std::make_unique()}; diff --git a/silkworm/db/mdbx_test.cpp b/silkworm/db/mdbx_test.cpp index 6336e1a724..d8adb0c743 100644 --- a/silkworm/db/mdbx_test.cpp +++ b/silkworm/db/mdbx_test.cpp @@ -91,7 +91,9 @@ static const std::map kGeneticCode{ {"UUU", "Phenylalanine"}, }; -namespace silkworm::db { +namespace silkworm::sw_mdbx { + +using namespace silkworm::db; TEST_CASE("Environment opening") { SECTION("Default page size on creation") { @@ -965,4 +967,4 @@ TEST_CASE("Multi-value erase+upsert w/ same value increases free pages") { } } -} // namespace silkworm::db +} // namespace silkworm::sw_mdbx diff --git a/silkworm/db/memory_mutation_cursor_test.cpp b/silkworm/db/memory_mutation_cursor_test.cpp index b7eb9373cf..d9200d501a 100644 --- a/silkworm/db/memory_mutation_cursor_test.cpp +++ b/silkworm/db/memory_mutation_cursor_test.cpp @@ -20,7 +20,9 @@ #include #include -namespace silkworm::db { +namespace silkworm::sw_mdbx { + +using namespace silkworm::db; static mdbx::env_managed create_main_env(EnvConfig& main_db_config) { auto main_env = open_env(main_db_config); @@ -1368,4 +1370,4 @@ TEST_CASE("MemoryMutationCursor: update value w/ different one", "[silkworm][nod #endif // SILKWORM_SANITIZE -} // namespace silkworm::db +} // namespace silkworm::sw_mdbx diff --git a/silkworm/db/prune_mode.cpp b/silkworm/db/prune_mode.cpp index 6280fdbde7..5295e320fc 100644 --- a/silkworm/db/prune_mode.cpp +++ b/silkworm/db/prune_mode.cpp @@ -23,6 +23,9 @@ namespace silkworm::db { +using sw_mdbx::from_slice; +using sw_mdbx::to_slice; + //! \brief Retrieves the proper BlockAmount prune threshold for given key static BlockAmount read_block_amount_for_key(mdbx::cursor& source, const char* key) { std::string key_str{key}; @@ -139,7 +142,7 @@ std::string PruneMode::to_string() const { } PruneMode read_prune_mode(mdbx::txn& txn) { - auto src{db::open_cursor(txn, table::kDatabaseInfo)}; + auto src = sw_mdbx::open_cursor(txn, table::kDatabaseInfo); auto history{read_block_amount_for_key(src, kPruneModeHistoryKey)}; auto receipts{read_block_amount_for_key(src, kPruneModeReceiptsKey)}; @@ -150,7 +153,7 @@ PruneMode read_prune_mode(mdbx::txn& txn) { } void write_prune_mode(mdbx::txn& txn, const PruneMode& value) { - auto target{db::open_cursor(txn, table::kDatabaseInfo)}; + auto target = sw_mdbx::open_cursor(txn, table::kDatabaseInfo); write_block_amount_for_key(target, kPruneModeHistoryKey, value.history()); write_block_amount_for_key(target, kPruneModeReceiptsKey, value.receipts()); write_block_amount_for_key(target, kPruneModeSendersKey, value.senders()); diff --git a/silkworm/db/snapshot_sync.cpp b/silkworm/db/snapshot_sync.cpp index d0734b123b..a9f87b75cd 100644 --- a/silkworm/db/snapshot_sync.cpp +++ b/silkworm/db/snapshot_sync.cpp @@ -65,7 +65,7 @@ SnapshotSync::SnapshotSync( ChainId chain_id, db::DataStoreRef data_store, std::filesystem::path tmp_dir_path, - stagedsync::StageScheduler& stage_scheduler) + datastore::StageScheduler& stage_scheduler) : settings_{std::move(settings)}, snapshots_config_{Config::lookup_known_config(chain_id, snapshot_file_is_fully_merged)}, data_store_{std::move(data_store)}, @@ -138,7 +138,7 @@ Task SnapshotSync::setup() { blocks_repository().reopen_folder(); // Update chain and stage progresses in database according to available snapshots - RWTxnManaged rw_txn = data_store_.chaindata.start_rw_tx(); + sw_mdbx::RWTxnManaged rw_txn = data_store_.chaindata.start_rw_tx(); update_database(rw_txn, blocks_repository().max_block_available(), [this] { return is_stopping_latch_.try_wait(); }); rw_txn.commit_and_stop(); @@ -337,7 +337,7 @@ void SnapshotSync::update_block_headers(RWTxn& txn, BlockNum max_block_available SILK_INFO << "SnapshotSync: database update started"; // Iterate on block header snapshots and write header-related tables - etl_mdbx::Collector hash_to_block_num_collector; + sw_mdbx::Collector hash_to_block_num_collector; intx::uint256 total_difficulty{0}; uint64_t block_count{0}; @@ -370,7 +370,7 @@ void SnapshotSync::update_block_headers(RWTxn& txn, BlockNum max_block_available } } - PooledCursor header_numbers_cursor{txn, table::kHeaderNumbers}; + sw_mdbx::PooledCursor header_numbers_cursor{txn, table::kHeaderNumbers}; hash_to_block_num_collector.load(header_numbers_cursor); SILK_INFO << "SnapshotSync: database table HeaderNumbers updated"; diff --git a/silkworm/db/snapshot_sync.hpp b/silkworm/db/snapshot_sync.hpp index 1fd1127b2f..c9e8d38718 100644 --- a/silkworm/db/snapshot_sync.hpp +++ b/silkworm/db/snapshot_sync.hpp @@ -53,7 +53,7 @@ class SnapshotSync { ChainId chain_id, db::DataStoreRef data_store, std::filesystem::path tmp_dir_path, - stagedsync::StageScheduler& stage_scheduler); + datastore::StageScheduler& stage_scheduler); Task run(); @@ -86,7 +86,7 @@ class SnapshotSync { snapshots::bittorrent::BitTorrentClient client_; db::Freezer snapshot_freezer_; - db::SnapshotMerger snapshot_merger_; + datastore::SnapshotMerger snapshot_merger_; std::latch is_stopping_latch_; std::atomic_bool setup_done_; diff --git a/silkworm/db/snapshot_sync_test.cpp b/silkworm/db/snapshot_sync_test.cpp index e4b062ac3e..a1f07af7ea 100644 --- a/silkworm/db/snapshot_sync_test.cpp +++ b/silkworm/db/snapshot_sync_test.cpp @@ -40,7 +40,7 @@ struct SettingsOverrides { bool no_downloader{false}; }; -class NoopStageSchedulerAdapter : public stagedsync::StageScheduler { +class NoopStageSchedulerAdapter : public datastore::StageScheduler { public: explicit NoopStageSchedulerAdapter() = default; ~NoopStageSchedulerAdapter() override = default; diff --git a/silkworm/db/stage.cpp b/silkworm/db/stage.cpp index 6648b81365..555c83cea3 100644 --- a/silkworm/db/stage.cpp +++ b/silkworm/db/stage.cpp @@ -22,23 +22,25 @@ namespace silkworm::stagedsync { +using namespace silkworm::db::stages; + Stage::Stage(SyncContext* sync_context, const char* stage_name) : sync_context_{sync_context}, stage_name_{stage_name} {} BlockNum Stage::get_progress(db::ROTxn& txn) { - return db::stages::read_stage_progress(txn, stage_name_); + return read_stage_progress(txn, stage_name_); } BlockNum Stage::get_prune_progress(db::ROTxn& txn) { - return db::stages::read_stage_prune_progress(txn, stage_name_); + return read_stage_prune_progress(txn, stage_name_); } void Stage::set_prune_progress(db::RWTxn& txn, BlockNum progress) { - db::stages::write_stage_prune_progress(txn, stage_name_, progress); + write_stage_prune_progress(txn, stage_name_, progress); } void Stage::update_progress(db::RWTxn& txn, BlockNum progress) { - db::stages::write_stage_progress(txn, stage_name_, progress); + write_stage_progress(txn, stage_name_, progress); } void Stage::check_block_sequence(BlockNum actual, BlockNum expected) { diff --git a/silkworm/db/stage.hpp b/silkworm/db/stage.hpp index 9a1077859d..38010d1266 100644 --- a/silkworm/db/stage.hpp +++ b/silkworm/db/stage.hpp @@ -27,6 +27,12 @@ #include #include +namespace silkworm::db { +using silkworm::sw_mdbx::MapConfig; +using silkworm::sw_mdbx::ROTxn; +using silkworm::sw_mdbx::RWTxn; +} // namespace silkworm::db + namespace silkworm::stagedsync { class StageError; diff --git a/silkworm/db/stages.cpp b/silkworm/db/stages.cpp index 1e70149c06..f64fbc23ec 100644 --- a/silkworm/db/stages.cpp +++ b/silkworm/db/stages.cpp @@ -22,7 +22,9 @@ namespace silkworm::db::stages { -static BlockNum get_stage_data(ROTxn& txn, const char* stage_name, const db::MapConfig& domain, +using namespace sw_mdbx; + +static BlockNum get_stage_data(ROTxn& txn, const char* stage_name, const MapConfig& domain, const char* key_prefix = nullptr) { if (!is_known_stage(stage_name)) { throw std::invalid_argument("Unknown stage name " + std::string(stage_name)); @@ -48,7 +50,7 @@ static BlockNum get_stage_data(ROTxn& txn, const char* stage_name, const db::Map } } -static void set_stage_data(RWTxn& txn, const char* stage_name, uint64_t block_num, const db::MapConfig& domain, +static void set_stage_data(RWTxn& txn, const char* stage_name, uint64_t block_num, const MapConfig& domain, const char* key_prefix = nullptr) { if (!is_known_stage(stage_name)) { throw std::invalid_argument("Unknown stage name"); @@ -63,7 +65,7 @@ static void set_stage_data(RWTxn& txn, const char* stage_name, uint64_t block_nu endian::store_big_u64(stage_progress.data(), block_num); auto target = txn.rw_cursor(domain); mdbx::slice key(item_key.c_str()); - mdbx::slice value{db::to_slice(stage_progress)}; + mdbx::slice value = sw_mdbx::to_slice(stage_progress); target->upsert(key, value); } catch (const mdbx::exception& ex) { std::string what("Error in " + std::string(__FUNCTION__) + " " + std::string(ex.what())); diff --git a/silkworm/db/stages.hpp b/silkworm/db/stages.hpp index 6b5c57945e..dab946ed9c 100644 --- a/silkworm/db/stages.hpp +++ b/silkworm/db/stages.hpp @@ -25,6 +25,9 @@ namespace silkworm::db::stages { +using sw_mdbx::ROTxn; +using sw_mdbx::RWTxn; + //! \brief Headers are downloaded, their Proof-Of-Work validity and chaining is verified inline constexpr const char* kHeadersKey{"Headers"}; diff --git a/silkworm/db/tables.cpp b/silkworm/db/tables.cpp index 64a80249da..f4f556f2f9 100644 --- a/silkworm/db/tables.cpp +++ b/silkworm/db/tables.cpp @@ -24,7 +24,7 @@ namespace silkworm::db::table { void check_or_create_chaindata_tables(RWTxn& txn) { for (const auto& config : kChainDataTables) { - if (db::has_map(txn, config.name)) { + if (has_map(txn, config.name)) { auto table_map{txn->open_map(config.name)}; auto table_info{txn->get_handle_info(table_map)}; auto table_key_mode{table_info.key_mode()}; @@ -48,7 +48,7 @@ void check_or_create_chaindata_tables(RWTxn& txn) { } } -std::optional get_map_config(const std::string& map_name) { +std::optional get_map_config(const std::string& map_name) { for (const auto& table_config : kChainDataTables) { if (table_config.name == map_name) { return table_config; diff --git a/silkworm/db/tables.hpp b/silkworm/db/tables.hpp index 7045c2b9b5..5d9f37660c 100644 --- a/silkworm/db/tables.hpp +++ b/silkworm/db/tables.hpp @@ -25,6 +25,8 @@ namespace silkworm::db::table { +using sw_mdbx::MapConfig; + //! Database schema version for compatibility w/ Erigon //! 5.0 - BlockTransaction table has canonical IDs (txs of non-canonical blocks moved to NonCanonicalTransaction table) //! 6.0 - BlockTransaction table has system-txs before/after each block (absent if block has no system-tx, but sequence increasing) @@ -414,7 +416,7 @@ inline constexpr MapConfig kChainDataTables[]{ }; //! \brief Ensures all defined tables are present in db with consistent flags. Should a table not exist it gets created -void check_or_create_chaindata_tables(RWTxn& txn); +void check_or_create_chaindata_tables(sw_mdbx::RWTxn& txn); //! \brief Get the table config associated to the table name (if any) std::optional get_map_config(const std::string& map_name); diff --git a/silkworm/db/test_util/mock_ro_cursor.hpp b/silkworm/db/test_util/mock_ro_cursor.hpp index 6580ece45c..996a3a940e 100644 --- a/silkworm/db/test_util/mock_ro_cursor.hpp +++ b/silkworm/db/test_util/mock_ro_cursor.hpp @@ -22,9 +22,12 @@ namespace silkworm::db::test_util { -class MockROCursor : public ROCursor { +class MockROCursor : public sw_mdbx::ROCursor { public: - MOCK_METHOD((void), bind, (ROTxn&, const MapConfig&), (override)); + using CursorResult = sw_mdbx::CursorResult; + using Slice = sw_mdbx::Slice; + + MOCK_METHOD((void), bind, (sw_mdbx::ROTxn&, const sw_mdbx::MapConfig&), (override)); MOCK_METHOD((size_t), size, (), (const, override)); MOCK_METHOD((bool), empty, (), (const)); MOCK_METHOD((bool), is_multi_value, (), (const, override)); @@ -44,8 +47,8 @@ class MockROCursor : public ROCursor { MOCK_METHOD((CursorResult), find, (const Slice&, bool), (override)); MOCK_METHOD((CursorResult), lower_bound, (const Slice&), (override)); MOCK_METHOD((CursorResult), lower_bound, (const Slice&, bool), (override)); - MOCK_METHOD((MoveResult), move, (MoveOperation, bool), (override)); - MOCK_METHOD((MoveResult), move, (MoveOperation, const Slice&, bool), (override)); + MOCK_METHOD((sw_mdbx::MoveResult), move, (sw_mdbx::MoveOperation, bool), (override)); + MOCK_METHOD((sw_mdbx::MoveResult), move, (sw_mdbx::MoveOperation, const Slice&, bool), (override)); MOCK_METHOD((bool), seek, (const Slice&), (override)); MOCK_METHOD((bool), eof, (), (const, override)); MOCK_METHOD((bool), on_first, (), (const, override)); diff --git a/silkworm/db/test_util/mock_txn.hpp b/silkworm/db/test_util/mock_txn.hpp index 01466048b3..9b85116891 100644 --- a/silkworm/db/test_util/mock_txn.hpp +++ b/silkworm/db/test_util/mock_txn.hpp @@ -22,14 +22,14 @@ namespace silkworm::db::test_util { -class MockROTxn : public ROTxn { +class MockROTxn : public sw_mdbx::ROTxn { public: - explicit MockROTxn() : ROTxn(txn_) {} + explicit MockROTxn() : sw_mdbx::ROTxn(txn_) {} MOCK_METHOD((bool), is_open, (), (const, override)); MOCK_METHOD((mdbx::env), db, (), (const, override)); - MOCK_METHOD((std::unique_ptr), ro_cursor, (const MapConfig&), (override)); - MOCK_METHOD((std::unique_ptr), ro_cursor_dup_sort, (const MapConfig&), (override)); + MOCK_METHOD((std::unique_ptr), ro_cursor, (const sw_mdbx::MapConfig&), (override)); + MOCK_METHOD((std::unique_ptr), ro_cursor_dup_sort, (const sw_mdbx::MapConfig&), (override)); MOCK_METHOD((void), abort, (), (override)); private: diff --git a/silkworm/db/test_util/temp_chain_data.cpp b/silkworm/db/test_util/temp_chain_data.cpp index 13d097a170..c1e004eb79 100644 --- a/silkworm/db/test_util/temp_chain_data.cpp +++ b/silkworm/db/test_util/temp_chain_data.cpp @@ -24,10 +24,12 @@ namespace silkworm::db::test_util { +using namespace sw_mdbx; + TempChainData::TempChainData(bool with_create_tables, bool in_memory) : data_dir_(tmp_dir_.path(), /*create=*/true), chain_config_(kMainnetConfig), - chaindata_env_config_(db::EnvConfig{ + chaindata_env_config_(EnvConfig{ .path = data_dir_.chaindata().path().string(), .create = true, .readonly = false, @@ -36,8 +38,8 @@ TempChainData::TempChainData(bool with_create_tables, bool in_memory) }) { chain_config_.genesis_hash.emplace(kMainnetGenesisHash); - env_ = std::make_unique(db::open_env(chaindata_env_config_)); - txn_ = std::make_unique(chaindata_rw().start_rw_tx()); + env_ = std::make_unique(open_env(chaindata_env_config_)); + txn_ = std::make_unique(chaindata_rw().start_rw_tx()); if (with_create_tables) { db::table::check_or_create_chaindata_tables(*txn_); diff --git a/silkworm/db/test_util/temp_chain_data.hpp b/silkworm/db/test_util/temp_chain_data.hpp index 0ae49f07d8..1c06a26546 100644 --- a/silkworm/db/test_util/temp_chain_data.hpp +++ b/silkworm/db/test_util/temp_chain_data.hpp @@ -47,13 +47,13 @@ class TempChainData { const DataDirectory& dir() const { return data_dir_; } - const db::EnvConfig& chaindata_env_config() const { return chaindata_env_config_; } + const sw_mdbx::EnvConfig& chaindata_env_config() const { return chaindata_env_config_; } - virtual db::ROAccess chaindata() const { - return db::ROAccess{*env_}; + virtual sw_mdbx::ROAccess chaindata() const { + return sw_mdbx::ROAccess{*env_}; } - virtual db::RWAccess chaindata_rw() const { - return db::RWAccess{*env_}; + virtual sw_mdbx::RWAccess chaindata_rw() const { + return sw_mdbx::RWAccess{*env_}; } mdbx::txn& txn() const { return *txn_; } @@ -77,7 +77,7 @@ class TempChainData { TemporaryDirectory tmp_dir_; DataDirectory data_dir_; ChainConfig chain_config_; - db::EnvConfig chaindata_env_config_; + sw_mdbx::EnvConfig chaindata_env_config_; std::unique_ptr env_; std::unique_ptr txn_; db::PruneMode prune_mode_; @@ -101,10 +101,10 @@ class TempChainDataStore : public TempChainData { db::DataStore& operator*() { return data_store_; } db::DataStore* operator->() { return &data_store_; } - db::ROAccess chaindata() const override { + sw_mdbx::ROAccess chaindata() const override { return data_store_.chaindata(); } - db::RWAccess chaindata_rw() const override { + sw_mdbx::RWAccess chaindata_rw() const override { return data_store_.chaindata_rw(); } diff --git a/silkworm/db/test_util/test_database_context.cpp b/silkworm/db/test_util/test_database_context.cpp index 26985ff05d..60112b7702 100644 --- a/silkworm/db/test_util/test_database_context.cpp +++ b/silkworm/db/test_util/test_database_context.cpp @@ -27,6 +27,8 @@ namespace silkworm::db::test_util { +using namespace sw_mdbx; + std::filesystem::path get_tests_dir() { auto working_dir = std::filesystem::current_path(); @@ -77,7 +79,7 @@ InMemoryState populate_genesis(RWTxn& txn, const std::filesystem::path& tests_di const uint8_t genesis_null_receipts[] = {0xf6}; // <- cbor encoded open_cursor(txn, table::kBlockReceipts) - .upsert(to_slice(block_hash_key).safe_middle(0, 8), to_slice(Bytes(genesis_null_receipts, 1))); + .upsert(sw_mdbx::to_slice(block_hash_key).safe_middle(0, 8), sw_mdbx::to_slice(Bytes(genesis_null_receipts, 1))); // Write Chain Settings auto config_data{genesis_json["config"].dump()}; diff --git a/silkworm/db/test_util/test_database_context.hpp b/silkworm/db/test_util/test_database_context.hpp index 2e90b73a1d..d2acf19dbb 100644 --- a/silkworm/db/test_util/test_database_context.hpp +++ b/silkworm/db/test_util/test_database_context.hpp @@ -39,11 +39,11 @@ class TestDatabaseContext { explicit TestDatabaseContext(const TemporaryDirectory& tmp_dir); virtual ~TestDatabaseContext() = default; - virtual db::ROAccess chaindata() const { - return db::ROAccess{*env_}; + virtual sw_mdbx::ROAccess chaindata() const { + return sw_mdbx::ROAccess{*env_}; } - virtual db::RWAccess chaindata_rw() const { - return db::RWAccess{*env_}; + virtual sw_mdbx::RWAccess chaindata_rw() const { + return sw_mdbx::RWAccess{*env_}; } silkworm::ChainConfig get_chain_config() const; @@ -76,10 +76,10 @@ class TestDataStore : public TestDatabaseContext { db::DataStore& operator*() { return data_store_; } db::DataStore* operator->() { return &data_store_; } - db::ROAccess chaindata() const override { + sw_mdbx::ROAccess chaindata() const override { return data_store_.chaindata(); } - db::RWAccess chaindata_rw() const override { + sw_mdbx::RWAccess chaindata_rw() const override { return data_store_.chaindata_rw(); } diff --git a/silkworm/db/util.cpp b/silkworm/db/util.cpp index fd32b18a25..574dd76e31 100644 --- a/silkworm/db/util.cpp +++ b/silkworm/db/util.cpp @@ -27,6 +27,9 @@ namespace silkworm::db { +using sw_mdbx::from_slice; +using sw_mdbx::to_slice; + Bytes storage_prefix(ByteView address, uint64_t incarnation) { SILKWORM_ASSERT(address.length() == kAddressLength || address.length() == kHashLength); Bytes res(address.length() + kIncarnationLength, '\0'); @@ -170,7 +173,7 @@ std::pair changeset_to_plainstate_format(const ByteView key, ByteV throw std::runtime_error("Invalid key length " + std::to_string(key.length()) + " in " + std::string(__FUNCTION__)); } -std::optional find_value_suffix(ROCursorDupSort& table, ByteView key, ByteView value_prefix) { +std::optional find_value_suffix(sw_mdbx::ROCursorDupSort& table, ByteView key, ByteView value_prefix) { auto value_prefix_slice{to_slice(value_prefix)}; auto data{table.lower_bound_multivalue(to_slice(key), value_prefix_slice, /*throw_notfound=*/false)}; if (!data || !data.value.starts_with(value_prefix_slice)) { @@ -182,7 +185,7 @@ std::optional find_value_suffix(ROCursorDupSort& table, ByteView key, return res; } -void upsert_storage_value(RWCursorDupSort& state_cursor, ByteView storage_prefix, ByteView location, ByteView new_value) { +void upsert_storage_value(sw_mdbx::RWCursorDupSort& state_cursor, ByteView storage_prefix, ByteView location, ByteView new_value) { if (find_value_suffix(state_cursor, storage_prefix, location)) { state_cursor.erase(); } diff --git a/silkworm/db/util.hpp b/silkworm/db/util.hpp index 7355cb25c6..a39a66fee9 100644 --- a/silkworm/db/util.hpp +++ b/silkworm/db/util.hpp @@ -124,20 +124,20 @@ std::tuple split_log_topic_key(const mdbx::slice& key); std::pair changeset_to_plainstate_format(ByteView key, ByteView value); inline mdbx::slice to_slice(const evmc::bytes32& value) { - return to_slice(ByteView{value.bytes}); + return sw_mdbx::to_slice(ByteView{value.bytes}); } inline mdbx::slice to_slice(const evmc::address& address) { - return to_slice(ByteView{address.bytes}); + return sw_mdbx::to_slice(ByteView{address.bytes}); } // If there exists an entry in a multivalue table with a given key and a value starting with a given prefix, // return the suffix of the value. // Otherwise, return nullopt. -std::optional find_value_suffix(ROCursorDupSort& table, ByteView key, ByteView value_prefix); +std::optional find_value_suffix(sw_mdbx::ROCursorDupSort& table, ByteView key, ByteView value_prefix); // We can't simply call upsert for storage values because they live in mdbx::value_mode::multi tables -void upsert_storage_value(RWCursorDupSort& state_cursor, ByteView storage_prefix, ByteView location, ByteView new_value); +void upsert_storage_value(sw_mdbx::RWCursorDupSort& state_cursor, ByteView storage_prefix, ByteView location, ByteView new_value); //! Build key for account domain given the target address and location Bytes account_domain_key(const evmc::address& address); diff --git a/silkworm/execution/local_state.hpp b/silkworm/execution/local_state.hpp index d878457eeb..0c5df81ae6 100644 --- a/silkworm/execution/local_state.hpp +++ b/silkworm/execution/local_state.hpp @@ -92,7 +92,7 @@ class LocalState : public State { private: BlockNum block_num_; - mutable db::ROTxnManaged txn_; + mutable sw_mdbx::ROTxnManaged txn_; db::DataModel data_model_; }; diff --git a/silkworm/node/CMakeLists.txt b/silkworm/node/CMakeLists.txt index 77df74ed02..d492647054 100644 --- a/silkworm/node/CMakeLists.txt +++ b/silkworm/node/CMakeLists.txt @@ -47,7 +47,7 @@ set(SILKWORM_NODE_PRIVATE_LIBS evmone magic_enum::magic_enum Microsoft.GSL::GSL - silkworm_db_etl + silkworm_etl silkworm_execution silkworm_interfaces silkworm_stages diff --git a/silkworm/node/backend/ethereum_backend.cpp b/silkworm/node/backend/ethereum_backend.cpp index 11398c2a53..f34bdb0554 100644 --- a/silkworm/node/backend/ethereum_backend.cpp +++ b/silkworm/node/backend/ethereum_backend.cpp @@ -20,7 +20,7 @@ namespace silkworm { EthereumBackEnd::EthereumBackEnd( const NodeSettings& node_settings, - db::ROAccess chaindata, + sw_mdbx::ROAccess chaindata, std::shared_ptr sentry_client) : EthereumBackEnd{ node_settings, @@ -32,7 +32,7 @@ EthereumBackEnd::EthereumBackEnd( EthereumBackEnd::EthereumBackEnd( const NodeSettings& node_settings, - db::ROAccess chaindata, + sw_mdbx::ROAccess chaindata, std::shared_ptr sentry_client, std::unique_ptr state_change_collection) : node_settings_(node_settings), diff --git a/silkworm/node/backend/ethereum_backend.hpp b/silkworm/node/backend/ethereum_backend.hpp index b0cdd2abc5..e04419a659 100644 --- a/silkworm/node/backend/ethereum_backend.hpp +++ b/silkworm/node/backend/ethereum_backend.hpp @@ -35,14 +35,14 @@ class EthereumBackEnd { public: explicit EthereumBackEnd( const NodeSettings& node_settings, - db::ROAccess chaindata, + sw_mdbx::ROAccess chaindata, std::shared_ptr sentry_client); ~EthereumBackEnd(); EthereumBackEnd(const EthereumBackEnd&) = delete; EthereumBackEnd& operator=(const EthereumBackEnd&) = delete; - db::ROAccess chaindata() const noexcept { return chaindata_; } + sw_mdbx::ROAccess chaindata() const noexcept { return chaindata_; } const std::string& node_name() const noexcept { return node_name_; } std::optional chain_id() const noexcept { return chain_id_; } std::optional etherbase() const noexcept { return node_settings_.etherbase; } @@ -57,13 +57,13 @@ class EthereumBackEnd { //! Constructor for testability EthereumBackEnd( const NodeSettings& node_settings, - db::ROAccess chaindata, + sw_mdbx::ROAccess chaindata, std::shared_ptr sentry_client, std::unique_ptr state_change_collection); private: const NodeSettings& node_settings_; - db::ROAccess chaindata_; + sw_mdbx::ROAccess chaindata_; std::string node_name_{kDefaultNodeName}; std::optional chain_id_{std::nullopt}; std::shared_ptr sentry_client_; diff --git a/silkworm/node/backend/ethereum_backend_test.cpp b/silkworm/node/backend/ethereum_backend_test.cpp index 0181ad90af..5080e68035 100644 --- a/silkworm/node/backend/ethereum_backend_test.cpp +++ b/silkworm/node/backend/ethereum_backend_test.cpp @@ -31,11 +31,11 @@ TEST_CASE("EthereumBackEnd", "[silkworm][backend][ethereum_backend]") { DataDirectory data_dir{tmp_dir.path()}; REQUIRE_NOTHROW(data_dir.deploy()); - db::EnvConfig db_config{data_dir.chaindata().path().string()}; + sw_mdbx::EnvConfig db_config{data_dir.chaindata().path().string()}; db_config.create = true; db_config.in_memory = true; - auto chaindata_env = db::open_env(db_config); - db::ROAccess chaindata{chaindata_env}; + auto chaindata_env = sw_mdbx::open_env(db_config); + sw_mdbx::ROAccess chaindata{chaindata_env}; NodeSettings node_settings; diff --git a/silkworm/node/common/node_settings.hpp b/silkworm/node/common/node_settings.hpp index c650d7d924..59933d43d2 100644 --- a/silkworm/node/common/node_settings.hpp +++ b/silkworm/node/common/node_settings.hpp @@ -34,7 +34,7 @@ namespace silkworm { struct NodeSettings { ApplicationInfo build_info; // Application build info (human-readable) std::unique_ptr data_directory; // Pointer to data folder - db::EnvConfig chaindata_env_config{}; // Chaindata db config + sw_mdbx::EnvConfig chaindata_env_config; // Chaindata db config uint64_t network_id{kMainnetConfig.chain_id}; // Network/Chain id std::optional chain_config; // Chain config size_t batch_size{512_Mebi}; // Batch size to use in stages @@ -49,7 +49,7 @@ struct NodeSettings { bool keep_db_txn_open{true}; // Whether to keep db transaction open between requests std::optional exec_api_address; // Execution API GRPC server bind address (IP:port) - db::etl::CollectorSettings etl() const { + etl::CollectorSettings etl() const { return {data_directory->temp().path(), etl_buffer_size}; } }; diff --git a/silkworm/node/execution/header_chain_plus_exec_test.cpp b/silkworm/node/execution/header_chain_plus_exec_test.cpp index edbe2dd4b0..29c06216dd 100644 --- a/silkworm/node/execution/header_chain_plus_exec_test.cpp +++ b/silkworm/node/execution/header_chain_plus_exec_test.cpp @@ -86,7 +86,7 @@ TEST_CASE("Headers receiving and saving") { db::DataModelFactory data_model_factory = context.data_model_factory(); NodeSettings node_settings = node::test_util::make_node_settings_from_temp_chain_data(context); - RWAccess db_access = context.chaindata_rw(); + auto db_access = context.chaindata_rw(); // creating the ExecutionEngine ExecutionEngineForTest exec_engine{ diff --git a/silkworm/node/node.cpp b/silkworm/node/node.cpp index f333cc11d7..227c4589ae 100644 --- a/silkworm/node/node.cpp +++ b/silkworm/node/node.cpp @@ -107,7 +107,7 @@ class NodeImpl final { std::unique_ptr bittorrent_client_; }; -static db::EnvConfig init_chain_data_db(NodeSettings& node_settings) { +static sw_mdbx::EnvConfig init_chain_data_db(NodeSettings& node_settings) { node_settings.data_directory->deploy(); node_settings.chain_config = db::chain_data_init(db::ChainDataInitSettings{ .chaindata_env_config = node_settings.chaindata_env_config, @@ -174,7 +174,7 @@ static stagedsync::StageContainerFactory make_stages_factory( } static sentry::SessionSentryClient::StatusDataProvider make_sentry_eth_status_data_provider( - db::ROAccess db_access, + sw_mdbx::ROAccess db_access, const ChainConfig& chain_config) { auto chain_head_provider = [db_access = std::move(db_access)] { return db::read_chain_head(db_access); diff --git a/silkworm/node/remote/ethbackend/grpc/server/backend_server_test.cpp b/silkworm/node/remote/ethbackend/grpc/server/backend_server_test.cpp index fdca9cba99..678f055b63 100644 --- a/silkworm/node/remote/ethbackend/grpc/server/backend_server_test.cpp +++ b/silkworm/node/remote/ethbackend/grpc/server/backend_server_test.cpp @@ -181,10 +181,11 @@ class MockSentryClient // TODO(canepat): better copy grpc_pick_unused_port_or_die to generate unused port const std::string kTestAddressUri{"localhost:12345"}; -const silkworm::db::MapConfig kTestMap{"TestTable"}; -const silkworm::db::MapConfig kTestMultiMap{"TestMultiTable", mdbx::key_mode::usual, mdbx::value_mode::multi}; +const silkworm::sw_mdbx::MapConfig kTestMap{"TestTable"}; +const silkworm::sw_mdbx::MapConfig kTestMultiMap{"TestMultiTable", mdbx::key_mode::usual, mdbx::value_mode::multi}; using namespace silkworm; +using namespace silkworm::sw_mdbx; using StateChangeTokenObserver = std::function)>; @@ -208,7 +209,7 @@ class TestableStateChangeCollection : public StateChangeCollection { class TestableEthereumBackEnd : public EthereumBackEnd { public: - TestableEthereumBackEnd(const NodeSettings& node_settings, db::ROAccess chaindata) + TestableEthereumBackEnd(const NodeSettings& node_settings, sw_mdbx::ROAccess chaindata) : EthereumBackEnd{ node_settings, std::move(chaindata), @@ -236,28 +237,28 @@ struct BackEndE2ETest { DataDirectory data_dir{tmp_dir.path()}; REQUIRE_NOTHROW(data_dir.deploy()); - db_config = std::make_unique(); + db_config = std::make_unique(); db_config->max_readers = options.chaindata_env_config.max_readers; db_config->path = data_dir.chaindata().path().string(); db_config->create = true; db_config->in_memory = true; - database_env = db::open_env(*db_config); + database_env = open_env(*db_config); auto rw_txn{database_env.start_write()}; - db::open_map(rw_txn, kTestMap); - db::open_map(rw_txn, kTestMultiMap); + open_map(rw_txn, kTestMap); + open_map(rw_txn, kTestMultiMap); rw_txn.commit(); - backend = std::make_unique(options, db::ROAccess{database_env}); + backend = std::make_unique(options, sw_mdbx::ROAccess{database_env}); server = std::make_unique(srv_config, *backend); server->build_and_start(); } void fill_tables() { auto rw_txn = database_env.start_write(); - db::PooledCursor rw_cursor1{rw_txn, kTestMap}; + PooledCursor rw_cursor1{rw_txn, kTestMap}; rw_cursor1.upsert(mdbx::slice{"AA"}, mdbx::slice{"00"}); rw_cursor1.upsert(mdbx::slice{"BB"}, mdbx::slice{"11"}); - db::PooledCursor rw_cursor2{rw_txn, kTestMultiMap}; + PooledCursor rw_cursor2{rw_txn, kTestMultiMap}; rw_cursor2.upsert(mdbx::slice{"AA"}, mdbx::slice{"00"}); rw_cursor2.upsert(mdbx::slice{"AA"}, mdbx::slice{"11"}); rw_cursor2.upsert(mdbx::slice{"AA"}, mdbx::slice{"22"}); @@ -267,9 +268,9 @@ struct BackEndE2ETest { void alter_tables() { auto rw_txn = database_env.start_write(); - db::PooledCursor rw_cursor1{rw_txn, kTestMap}; + PooledCursor rw_cursor1{rw_txn, kTestMap}; rw_cursor1.upsert(mdbx::slice{"CC"}, mdbx::slice{"22"}); - db::PooledCursor rw_cursor2{rw_txn, kTestMultiMap}; + PooledCursor rw_cursor2{rw_txn, kTestMultiMap}; rw_cursor2.upsert(mdbx::slice{"AA"}, mdbx::slice{"33"}); rw_cursor2.upsert(mdbx::slice{"BB"}, mdbx::slice{"33"}); rw_txn.commit(); @@ -285,7 +286,7 @@ struct BackEndE2ETest { std::unique_ptr backend_client; rpc::ServerSettings srv_config; TemporaryDirectory tmp_dir; - std::unique_ptr db_config; + std::unique_ptr db_config; mdbx::env_managed database_env; std::unique_ptr backend; std::unique_ptr server; @@ -304,12 +305,12 @@ TEST_CASE("BackEndServer", "[silkworm][node][rpc]") { TemporaryDirectory tmp_dir; DataDirectory data_dir{tmp_dir.path()}; REQUIRE_NOTHROW(data_dir.deploy()); - db::EnvConfig db_config{data_dir.chaindata().path().string()}; + EnvConfig db_config{data_dir.chaindata().path().string()}; db_config.create = true; db_config.in_memory = true; - auto chaindata_env = db::open_env(db_config); + auto chaindata_env = open_env(db_config); NodeSettings node_settings; - TestableEthereumBackEnd backend{node_settings, db::ROAccess{chaindata_env}}; + TestableEthereumBackEnd backend{node_settings, sw_mdbx::ROAccess{chaindata_env}}; SECTION("BackEndServer::BackEndServer OK: create/destroy server") { BackEndServer server{srv_config, backend}; diff --git a/silkworm/node/stagedsync/execution_engine.cpp b/silkworm/node/stagedsync/execution_engine.cpp index a279f32c14..2b675db4f2 100644 --- a/silkworm/node/stagedsync/execution_engine.cpp +++ b/silkworm/node/stagedsync/execution_engine.cpp @@ -36,7 +36,7 @@ ExecutionEngine::ExecutionEngine( db::DataModelFactory data_model_factory, std::optional log_timer_factory, StageContainerFactory stages_factory, - db::RWAccess dba) + sw_mdbx::RWAccess dba) : context_pool_{executor ? std::unique_ptr>{} : std::make_unique>(1)}, executor_{executor ? std::move(*executor) : context_pool_->any_executor()}, node_settings_{ns}, @@ -337,7 +337,7 @@ bool ExecutionEngine::is_canonical(Hash header_hash) const { return main_chain_.is_finalized_canonical(header_hash); } -StageScheduler& ExecutionEngine::stage_scheduler() const { +datastore::StageScheduler& ExecutionEngine::stage_scheduler() const { return main_chain_.stage_scheduler(); } diff --git a/silkworm/node/stagedsync/execution_engine.hpp b/silkworm/node/stagedsync/execution_engine.hpp index d058bdf491..ea822bf791 100644 --- a/silkworm/node/stagedsync/execution_engine.hpp +++ b/silkworm/node/stagedsync/execution_engine.hpp @@ -63,7 +63,7 @@ class ExecutionEngine : public execution::api::ExecutionEngine, public Stoppable db::DataModelFactory data_model_factory, std::optional log_timer_factory, StageContainerFactory stages_factory, - db::RWAccess dba); + sw_mdbx::RWAccess dba); ~ExecutionEngine() override = default; // needed to circumvent mdbx threading model limitations @@ -101,7 +101,7 @@ class ExecutionEngine : public execution::api::ExecutionEngine, public Stoppable std::vector get_last_headers(uint64_t limit) const override; std::optional get_header_td(Hash, std::optional) const override; - StageScheduler& stage_scheduler() const; + datastore::StageScheduler& stage_scheduler() const; protected: struct ForkingPath { diff --git a/silkworm/node/stagedsync/execution_engine_test.cpp b/silkworm/node/stagedsync/execution_engine_test.cpp index 664c45b470..ab63d15cd4 100644 --- a/silkworm/node/stagedsync/execution_engine_test.cpp +++ b/silkworm/node/stagedsync/execution_engine_test.cpp @@ -70,7 +70,7 @@ TEST_CASE("ExecutionEngine Integration Test", "[node][execution][execution_engin .keep_db_txn_open = true, }; - RWAccess db_access = db_context.chaindata_rw(); + auto db_access = db_context.chaindata_rw(); ExecutionEngineForTest exec_engine{ runner.executor(), diff --git a/silkworm/node/stagedsync/execution_pipeline.cpp b/silkworm/node/stagedsync/execution_pipeline.cpp index 12c4186e05..64c1f66b46 100644 --- a/silkworm/node/stagedsync/execution_pipeline.cpp +++ b/silkworm/node/stagedsync/execution_pipeline.cpp @@ -110,8 +110,8 @@ bool ExecutionPipeline::stop() { return stopped; } -StageScheduler& ExecutionPipeline::stage_scheduler() const { - return *dynamic_cast(stages_.at(kTriggersStageKey).get()); +datastore::StageScheduler& ExecutionPipeline::stage_scheduler() const { + return *dynamic_cast(stages_.at(kTriggersStageKey).get()); } Stage::Result ExecutionPipeline::forward(db::RWTxn& cycle_txn, BlockNum target_block_num) { diff --git a/silkworm/node/stagedsync/execution_pipeline.hpp b/silkworm/node/stagedsync/execution_pipeline.hpp index 6b698c7bb8..950ffa8b88 100644 --- a/silkworm/node/stagedsync/execution_pipeline.hpp +++ b/silkworm/node/stagedsync/execution_pipeline.hpp @@ -64,7 +64,7 @@ class ExecutionPipeline : public Stoppable { bool stop() override; - StageScheduler& stage_scheduler() const; + datastore::StageScheduler& stage_scheduler() const; private: db::DataModelFactory data_model_factory_; diff --git a/silkworm/node/stagedsync/forks/extending_fork.cpp b/silkworm/node/stagedsync/forks/extending_fork.cpp index d680b376f4..3b3ea8eb9d 100644 --- a/silkworm/node/stagedsync/forks/extending_fork.cpp +++ b/silkworm/node/stagedsync/forks/extending_fork.cpp @@ -69,7 +69,7 @@ void ExtendingFork::start_with(BlockId new_head, std::list( forking_point_, - db::ROTxnManaged(main_chain_.tx().db()), + sw_mdbx::ROTxnManaged(main_chain_.tx().db()), main_chain_.data_model_factory(), main_chain_.log_timer_factory(), main_chain_.stages_factory(), diff --git a/silkworm/node/stagedsync/forks/fork.cpp b/silkworm/node/stagedsync/forks/fork.cpp index 056e39bda6..fc4af1cc2d 100644 --- a/silkworm/node/stagedsync/forks/fork.cpp +++ b/silkworm/node/stagedsync/forks/fork.cpp @@ -29,8 +29,8 @@ using execution::api::ValidationError; using execution::api::ValidChain; using execution::api::VerificationResult; -static db::MemoryOverlay create_memory_db(const std::filesystem::path& base_path, db::ROTxn& main_tx) { - db::MemoryOverlay memory_overlay{ +static sw_mdbx::MemoryOverlay create_memory_db(const std::filesystem::path& base_path, db::ROTxn& main_tx) { + sw_mdbx::MemoryOverlay memory_overlay{ TemporaryDirectory::get_unique_temporary_path(base_path), &main_tx, db::table::get_map_config, @@ -39,7 +39,7 @@ static db::MemoryOverlay create_memory_db(const std::filesystem::path& base_path // Create predefined tables for chaindata schema auto txn = memory_overlay.start_rw_txn(); - db::RWTxnUnmanaged txn_ref{txn}; + sw_mdbx::RWTxnUnmanaged txn_ref{txn}; db::table::check_or_create_chaindata_tables(txn_ref); txn.commit(); @@ -48,7 +48,7 @@ static db::MemoryOverlay create_memory_db(const std::filesystem::path& base_path Fork::Fork( BlockId forking_point, - db::ROTxnManaged main_tx, + sw_mdbx::ROTxnManaged main_tx, db::DataModelFactory data_model_factory, std::optional log_timer_factory, const StageContainerFactory& stages_factory, diff --git a/silkworm/node/stagedsync/forks/fork.hpp b/silkworm/node/stagedsync/forks/fork.hpp index 92e41aef46..d8cb35f096 100644 --- a/silkworm/node/stagedsync/forks/fork.hpp +++ b/silkworm/node/stagedsync/forks/fork.hpp @@ -40,7 +40,7 @@ class Fork { public: explicit Fork( BlockId forking_point, - db::ROTxnManaged main_tx, + sw_mdbx::ROTxnManaged main_tx, db::DataModelFactory data_model_factory, std::optional log_timer_factory, const StageContainerFactory& stages_factory, @@ -84,9 +84,9 @@ class Fork { std::set collect_bad_headers(execution::api::InvalidChain& invalid_chain); - db::ROTxnManaged main_tx_; - db::MemoryOverlay memory_db_; - mutable db::MemoryMutation memory_tx_; + sw_mdbx::ROTxnManaged main_tx_; + sw_mdbx::MemoryOverlay memory_db_; + mutable sw_mdbx::MemoryMutation memory_tx_; db::DataModelFactory data_model_factory_; ExecutionPipeline pipeline_; diff --git a/silkworm/node/stagedsync/forks/fork_test.cpp b/silkworm/node/stagedsync/forks/fork_test.cpp index 399474e369..43ada30516 100644 --- a/silkworm/node/stagedsync/forks/fork_test.cpp +++ b/silkworm/node/stagedsync/forks/fork_test.cpp @@ -66,7 +66,7 @@ TEST_CASE("Fork") { Environment::set_stop_before_stage(db::stages::kSendersKey); // only headers, block hashes and bodies NodeSettings node_settings = node::test_util::make_node_settings_from_temp_chain_data(context); - db::RWAccess db_access = context.chaindata_rw(); + sw_mdbx::RWAccess db_access = context.chaindata_rw(); MainChain main_chain{ ioc.get_executor(), @@ -127,7 +127,7 @@ TEST_CASE("Fork") { ForkForTest fork{ forking_point, - db::ROTxnManaged(main_chain.tx().db()), // this need to be on a different thread than main_chain + sw_mdbx::ROTxnManaged(main_chain.tx().db()), // this need to be on a different thread than main_chain data_model_factory, /* log_timer_factory = */ std::nullopt, main_chain.stages_factory(), diff --git a/silkworm/node/stagedsync/forks/main_chain.cpp b/silkworm/node/stagedsync/forks/main_chain.cpp index 1a8f9920fb..9085df75b0 100644 --- a/silkworm/node/stagedsync/forks/main_chain.cpp +++ b/silkworm/node/stagedsync/forks/main_chain.cpp @@ -32,7 +32,7 @@ namespace { //! @brief Handles the transaction lifecycle for long-standing and per-request transactions class TransactionHandler { public: - TransactionHandler(silkworm::db::RWTxnManaged& txn, silkworm::db::RWAccess& db_access, bool keep_db_txn_open = true) + TransactionHandler(silkworm::sw_mdbx::RWTxnManaged& txn, silkworm::sw_mdbx::RWAccess& db_access, bool keep_db_txn_open = true) : txn_{txn}, db_access_{db_access}, keep_db_txn_open_{keep_db_txn_open} { if (!keep_db_txn_open_) { if (request_count_ == 0 && !txn_.is_open()) { @@ -55,8 +55,8 @@ class TransactionHandler { } private: - silkworm::db::RWTxnManaged& txn_; - silkworm::db::RWAccess& db_access_; + silkworm::sw_mdbx::RWTxnManaged& txn_; + silkworm::sw_mdbx::RWAccess& db_access_; bool keep_db_txn_open_{true}; static inline SILKWORM_THREAD_LOCAL int request_count_; }; @@ -78,7 +78,7 @@ MainChain::MainChain( db::DataModelFactory data_model_factory, std::optional log_timer_factory, StageContainerFactory stages_factory, - db::RWAccess dba) + sw_mdbx::RWAccess dba) : executor_{std::move(executor)}, node_settings_{ns}, data_model_factory_{std::move(data_model_factory)}, @@ -154,7 +154,7 @@ const std::optional& MainChain::log_timer_factory() const { return log_timer_factory_; } -StageScheduler& MainChain::stage_scheduler() const { +datastore::StageScheduler& MainChain::stage_scheduler() const { return pipeline_.stage_scheduler(); } diff --git a/silkworm/node/stagedsync/forks/main_chain.hpp b/silkworm/node/stagedsync/forks/main_chain.hpp index 09fba2eed7..a9ee034c4d 100644 --- a/silkworm/node/stagedsync/forks/main_chain.hpp +++ b/silkworm/node/stagedsync/forks/main_chain.hpp @@ -50,7 +50,7 @@ class MainChain { db::DataModelFactory data_model_factory, std::optional log_timer_factory, StageContainerFactory stages_factory, - db::RWAccess dba); + sw_mdbx::RWAccess dba); void open(); // needed to circumvent mdbx threading model limitations void close(); @@ -100,7 +100,7 @@ class MainChain { const db::DataModelFactory& data_model_factory() const { return data_model_factory_; } const std::optional& log_timer_factory() const; const StageContainerFactory& stages_factory() const { return stages_factory_; } - StageScheduler& stage_scheduler() const; + datastore::StageScheduler& stage_scheduler() const; protected: db::DataModel data_model() const { return data_model_factory_(tx_); } @@ -119,8 +119,8 @@ class MainChain { db::DataModelFactory data_model_factory_; std::optional log_timer_factory_; StageContainerFactory stages_factory_; - mutable db::RWAccess db_access_; - mutable db::RWTxnManaged tx_; + mutable sw_mdbx::RWAccess db_access_; + mutable sw_mdbx::RWTxnManaged tx_; bool is_first_sync_{true}; ExecutionPipeline pipeline_; diff --git a/silkworm/node/stagedsync/forks/main_chain_test.cpp b/silkworm/node/stagedsync/forks/main_chain_test.cpp index 2925585693..03031d10bc 100644 --- a/silkworm/node/stagedsync/forks/main_chain_test.cpp +++ b/silkworm/node/stagedsync/forks/main_chain_test.cpp @@ -170,7 +170,7 @@ TEST_CASE("MainChain") { Environment::set_stop_before_stage(stages::kSendersKey); // only headers, block hashes and bodies NodeSettings node_settings = node::test_util::make_node_settings_from_temp_chain_data(context); - RWAccess db_access = context.chaindata_rw(); + auto db_access = context.chaindata_rw(); MainChainForTest main_chain{ ioc.get_executor(), node_settings, diff --git a/silkworm/node/stagedsync/stages/CMakeLists.txt b/silkworm/node/stagedsync/stages/CMakeLists.txt index 6738816a8c..44dfdceac8 100644 --- a/silkworm/node/stagedsync/stages/CMakeLists.txt +++ b/silkworm/node/stagedsync/stages/CMakeLists.txt @@ -30,7 +30,7 @@ set(LIBS_PUBLIC silkworm_block_execution silkworm_core silkworm_db - silkworm_db_etl + silkworm_etl silkworm_infra ) diff --git a/silkworm/node/stagedsync/stages/stage_blockhashes.cpp b/silkworm/node/stagedsync/stages/stage_blockhashes.cpp index ac59e2f51a..74c1035f1a 100644 --- a/silkworm/node/stagedsync/stages/stage_blockhashes.cpp +++ b/silkworm/node/stagedsync/stages/stage_blockhashes.cpp @@ -22,8 +22,12 @@ namespace silkworm::stagedsync { -using db::etl::Entry; -using db::etl_mdbx::Collector; +using etl::Entry; +using sw_mdbx::Collector; + +namespace db { + using namespace silkworm::db; +} Stage::Result BlockHashes::forward(db::RWTxn& txn) { /* @@ -167,7 +171,7 @@ void BlockHashes::collect_and_load(db::RWTxn& txn, const BlockNum from, const Bl auto expected_block_num{from + 1}; auto header_key{db::block_key(expected_block_num)}; auto canon_hashes_cursor = txn.rw_cursor(db::table::kCanonicalHashes); - auto data{canon_hashes_cursor->find(db::to_slice(header_key), /*throw_notfound=*/false)}; + auto data{canon_hashes_cursor->find(sw_mdbx::to_slice(header_key), /*throw_notfound=*/false)}; while (data.done) { reached_block_num_ = endian::load_big_u64(static_cast(data.key.data())); if (reached_block_num_ > to) { @@ -181,9 +185,9 @@ void BlockHashes::collect_and_load(db::RWTxn& txn, const BlockNum from, const Bl " expected " + std::to_string(kHashLength)); } - collector_->collect(Entry{Bytes{db::from_slice(data.value)}, operation_ == OperationType::kForward - ? Bytes{db::from_slice(data.key)} - : Bytes{}}); + collector_->collect(Entry{Bytes{sw_mdbx::from_slice(data.value)}, operation_ == OperationType::kForward + ? Bytes{sw_mdbx::from_slice(data.key)} + : Bytes{}}); // Do we need to abort ? if (auto now{std::chrono::steady_clock::now()}; log_time <= now) { diff --git a/silkworm/node/stagedsync/stages/stage_blockhashes.hpp b/silkworm/node/stagedsync/stages/stage_blockhashes.hpp index 822a61f3f9..ccc2787723 100644 --- a/silkworm/node/stagedsync/stages/stage_blockhashes.hpp +++ b/silkworm/node/stagedsync/stages/stage_blockhashes.hpp @@ -23,8 +23,8 @@ namespace silkworm::stagedsync { class BlockHashes final : public Stage { public: - explicit BlockHashes(SyncContext* sync_context, db::etl::CollectorSettings etl_settings) - : Stage(sync_context, db::stages::kBlockHashesKey), + explicit BlockHashes(SyncContext* sync_context, etl::CollectorSettings etl_settings) + : Stage(sync_context, silkworm::db::stages::kBlockHashesKey), etl_settings_(std::move(etl_settings)) {} BlockHashes(const BlockHashes&) = delete; // not copyable BlockHashes(BlockHashes&&) = delete; // nor movable @@ -36,8 +36,8 @@ class BlockHashes final : public Stage { std::vector get_log_progress() final; private: - db::etl::CollectorSettings etl_settings_; - std::unique_ptr collector_; + etl::CollectorSettings etl_settings_; + std::unique_ptr collector_; /* Stats */ std::atomic_uint32_t current_phase_{0}; diff --git a/silkworm/node/stagedsync/stages/stage_bodies_test.cpp b/silkworm/node/stagedsync/stages/stage_bodies_test.cpp index c461554b85..901b99ee4e 100644 --- a/silkworm/node/stagedsync/stages/stage_bodies_test.cpp +++ b/silkworm/node/stagedsync/stages/stage_bodies_test.cpp @@ -39,7 +39,7 @@ TEST_CASE("BodiesStage - data model") { context.add_genesis_data(); context.commit_txn(); - db::RWAccess chaindata = context.chaindata_rw(); + sw_mdbx::RWAccess chaindata = context.chaindata_rw(); auto data_model_factory = context.data_model_factory(); auto& chain_config = context.chain_config(); @@ -50,7 +50,7 @@ TEST_CASE("BodiesStage - data model") { * h0 <----- h1 */ SECTION("one invalid body after the genesis") { - db::RWTxnManaged tx = chaindata.start_rw_tx(); + auto tx = chaindata.start_rw_tx(); db::DataModel data_model = data_model_factory(tx); auto header0_hash = db::read_canonical_header_hash(tx, 0); @@ -91,7 +91,7 @@ TEST_CASE("BodiesStage - data model") { } SECTION("one valid body after the genesis") { - db::RWTxnManaged tx = chaindata.start_rw_tx(); + auto tx = chaindata.start_rw_tx(); db::DataModel data_model = data_model_factory(tx); auto header0_hash = db::read_canonical_header_hash(tx, 0); diff --git a/silkworm/node/stagedsync/stages/stage_call_trace_index.cpp b/silkworm/node/stagedsync/stages/stage_call_trace_index.cpp index 83b4f06df2..9b81ff4083 100644 --- a/silkworm/node/stagedsync/stages/stage_call_trace_index.cpp +++ b/silkworm/node/stagedsync/stages/stage_call_trace_index.cpp @@ -21,6 +21,7 @@ namespace silkworm::stagedsync { using namespace silkworm::db; +using namespace silkworm::sw_mdbx; CallTraceIndex::CallTraceIndex(SyncContext* sync_context, size_t batch_size, @@ -233,8 +234,6 @@ Stage::Result CallTraceIndex::prune(RWTxn& txn) { } void CallTraceIndex::forward_impl(RWTxn& txn, const BlockNum from, const BlockNum to) { - using etl_mdbx::Collector; - const MapConfig source_config{table::kCallTraceSet}; std::unique_lock log_lck(sl_mutex_); diff --git a/silkworm/node/stagedsync/stages/stage_call_trace_index.hpp b/silkworm/node/stagedsync/stages/stage_call_trace_index.hpp index afb063f1d3..f88368d348 100644 --- a/silkworm/node/stagedsync/stages/stage_call_trace_index.hpp +++ b/silkworm/node/stagedsync/stages/stage_call_trace_index.hpp @@ -29,7 +29,7 @@ class CallTraceIndex : public Stage { public: CallTraceIndex(SyncContext* sync_context, size_t batch_size, - db::etl::CollectorSettings etl_settings, + etl::CollectorSettings etl_settings, db::BlockAmount prune_mode); CallTraceIndex(const CallTraceIndex&) = delete; // not copyable CallTraceIndex(CallTraceIndex&&) = delete; // nor movable @@ -60,12 +60,12 @@ class CallTraceIndex : public Stage { void reset_log_progress(); // Clears out all logging vars size_t batch_size_; - db::etl::CollectorSettings etl_settings_; + etl::CollectorSettings etl_settings_; db::BlockAmount prune_mode_; - std::unique_ptr call_from_collector_; - std::unique_ptr call_to_collector_; - std::unique_ptr index_loader_; + std::unique_ptr call_from_collector_; + std::unique_ptr call_to_collector_; + std::unique_ptr index_loader_; //! Flag indicating if we're in ETL loading phase (for logging purposes) std::atomic_bool loading_{false}; diff --git a/silkworm/node/stagedsync/stages/stage_execution.cpp b/silkworm/node/stagedsync/stages/stage_execution.cpp index 239ab1d83d..80b59e3be3 100644 --- a/silkworm/node/stagedsync/stages/stage_execution.cpp +++ b/silkworm/node/stagedsync/stages/stage_execution.cpp @@ -34,6 +34,7 @@ namespace silkworm::stagedsync { using namespace silkworm::db; +using namespace silkworm::sw_mdbx; Stage::Result Execution::forward(RWTxn& txn) { Stage::Result ret{Stage::Result::kSuccess}; diff --git a/silkworm/node/stagedsync/stages/stage_execution.hpp b/silkworm/node/stagedsync/stages/stage_execution.hpp index 88d9e533f0..e1909cd3a8 100644 --- a/silkworm/node/stagedsync/stages/stage_execution.hpp +++ b/silkworm/node/stagedsync/stages/stage_execution.hpp @@ -74,12 +74,12 @@ class Execution final : public Stage { BlockNum prune_call_traces_threshold); //! \brief For given changeset cursor/bucket it reverts the changes on states buckets - static void unwind_state_from_changeset(db::ROCursor& source_changeset, db::RWCursorDupSort& plain_state_table, - db::RWCursor& plain_code_table, BlockNum unwind_to); + static void unwind_state_from_changeset(sw_mdbx::ROCursor& source_changeset, sw_mdbx::RWCursorDupSort& plain_state_table, + sw_mdbx::RWCursor& plain_code_table, BlockNum unwind_to); //! \brief Revert State for given address/storage location - static void revert_state(ByteView key, ByteView value, db::RWCursorDupSort& plain_state_table, - db::RWCursor& plain_code_table); + static void revert_state(ByteView key, ByteView value, sw_mdbx::RWCursorDupSort& plain_state_table, + sw_mdbx::RWCursor& plain_code_table); // Stats std::mutex progress_mtx_; // Synchronizes access to progress stats diff --git a/silkworm/node/stagedsync/stages/stage_hashstate.cpp b/silkworm/node/stagedsync/stages/stage_hashstate.cpp index 94fff21645..9c23aac70f 100644 --- a/silkworm/node/stagedsync/stages/stage_hashstate.cpp +++ b/silkworm/node/stagedsync/stages/stage_hashstate.cpp @@ -30,7 +30,9 @@ namespace silkworm::stagedsync { using namespace silkworm::db; -using silkworm::db::etl::Entry; +using silkworm::etl::Entry; +using sw_mdbx::from_slice; +using sw_mdbx::to_slice; Stage::Result HashState::forward(RWTxn& txn) { Stage::Result ret{Stage::Result::kSuccess}; @@ -63,7 +65,7 @@ Stage::Result HashState::forward(RWTxn& txn) { } reset_log_progress(); - collector_ = std::make_unique(etl_settings_); + collector_ = std::make_unique(etl_settings_); if (!previous_progress || segment_width > stages::kLargeBlockSegmentWorthRegen) { // Clear any previous contents @@ -289,10 +291,10 @@ Stage::Result HashState::hash_from_plainstate(RWTxn& txn) { throw std::runtime_error(std::string(table::kHashedStorage.name) + " should be empty"); // ETL key contains hashed location; for DB put we need to move it from key to value - const etl_mdbx::LoadFunc load_func = [&storage_target]( - const Entry& entry, - RWCursorDupSort& target, - MDBX_put_flags_t) -> void { + const sw_mdbx::LoadFunc load_func = [&storage_target]( + const Entry& entry, + sw_mdbx::RWCursorDupSort& target, + MDBX_put_flags_t) -> void { if (entry.value.empty()) { return; } @@ -466,7 +468,7 @@ Stage::Result HashState::hash_from_account_changeset(RWTxn& txn, BlockNum previo evmc::address address{bytes_to_address(changeset_value_view)}; if (!changed_addresses.contains(address)) { auto address_hash{to_bytes32(keccak256(address.bytes).bytes)}; - auto plainstate_data{source_plainstate->find(to_slice(address), /*throw_notfound=*/false)}; + auto plainstate_data{source_plainstate->find(db::to_slice(address), /*throw_notfound=*/false)}; if (plainstate_data.done) { Bytes current_value{from_slice(plainstate_data.value)}; changed_addresses[address] = std::make_pair(address_hash, current_value); @@ -826,7 +828,7 @@ Stage::Result HashState::write_changes_from_changed_addresses(RWTxn& txn, const auto& [address_hash, current_encoded_value] = pair; if (!current_encoded_value.empty()) { // Update HashedAccounts table - target_hashed_accounts->upsert(to_slice(address_hash), to_slice(current_encoded_value)); + target_hashed_accounts->upsert(db::to_slice(address_hash), to_slice(current_encoded_value)); // Lookup value in PlainCodeHash for Contract const auto incarnation{Account::incarnation_from_encoded_storage(current_encoded_value)}; @@ -845,7 +847,7 @@ Stage::Result HashState::write_changes_from_changed_addresses(RWTxn& txn, const } } } else { - target_hashed_accounts->erase(to_slice(address_hash)); + target_hashed_accounts->erase(db::to_slice(address_hash)); } } diff --git a/silkworm/node/stagedsync/stages/stage_hashstate.hpp b/silkworm/node/stagedsync/stages/stage_hashstate.hpp index d7c209c077..e776591f2d 100644 --- a/silkworm/node/stagedsync/stages/stage_hashstate.hpp +++ b/silkworm/node/stagedsync/stages/stage_hashstate.hpp @@ -25,7 +25,7 @@ class HashState final : public Stage { public: HashState( SyncContext* sync_context, - db::etl::CollectorSettings etl_settings) + etl::CollectorSettings etl_settings) : Stage(sync_context, db::stages::kHashStateKey), etl_settings_(std::move(etl_settings)) {} HashState(const HashState&) = delete; // not copyable @@ -70,7 +70,7 @@ class HashState final : public Stage { Stage::Result write_changes_from_changed_addresses(db::RWTxn& txn, const ChangedAddresses& changed_addresses); //! \brief Writes to db the changes collected from storage changeset scan either in forward or unwind mode - Stage::Result write_changes_from_changed_storage(db::RWTxn& txn, db::StorageChanges& storage_changes, + Stage::Result write_changes_from_changed_storage(db::RWTxn& txn, silkworm::db::StorageChanges& storage_changes, const absl::btree_map& hashed_addresses); //! \brief Resets all fields related to log progress tracking @@ -92,8 +92,8 @@ class HashState final : public Stage { std::string current_key_; // Collector (used only in !incremental_) - db::etl::CollectorSettings etl_settings_; - std::unique_ptr collector_; + etl::CollectorSettings etl_settings_; + std::unique_ptr collector_; }; } // namespace silkworm::stagedsync diff --git a/silkworm/node/stagedsync/stages/stage_headers_test.cpp b/silkworm/node/stagedsync/stages/stage_headers_test.cpp index 2295b5d78b..ac756a7058 100644 --- a/silkworm/node/stagedsync/stages/stage_headers_test.cpp +++ b/silkworm/node/stagedsync/stages/stage_headers_test.cpp @@ -39,7 +39,7 @@ TEST_CASE("HeadersStage - data model") { context.add_genesis_data(); context.commit_txn(); - RWAccess chaindata = context.chaindata_rw(); + auto chaindata = context.chaindata_rw(); auto data_model_factory = context.data_model_factory(); /* status: @@ -48,7 +48,7 @@ TEST_CASE("HeadersStage - data model") { * h0 <----- h1 */ SECTION("one header after the genesis") { - RWTxnManaged tx = chaindata.start_rw_tx(); + auto tx = chaindata.start_rw_tx(); DataModel data_model = data_model_factory(tx); auto header0_hash = read_canonical_header_hash(tx, 0); @@ -92,7 +92,7 @@ TEST_CASE("HeadersStage - data model") { * |-- h1' */ SECTION("some header after the genesis") { - RWTxnManaged tx = chaindata.start_rw_tx(); + auto tx = chaindata.start_rw_tx(); DataModel data_model = data_model_factory(tx); // starting from an initial status diff --git a/silkworm/node/stagedsync/stages/stage_history_index.cpp b/silkworm/node/stagedsync/stages/stage_history_index.cpp index 1fb90a7b56..15b30a0b43 100644 --- a/silkworm/node/stagedsync/stages/stage_history_index.cpp +++ b/silkworm/node/stagedsync/stages/stage_history_index.cpp @@ -24,6 +24,11 @@ namespace silkworm::stagedsync { using namespace silkworm::db; +using silkworm::sw_mdbx::from_slice; +using silkworm::sw_mdbx::to_slice; +namespace bitmap { + using namespace silkworm::sw_mdbx::bitmap; +} Stage::Result HistoryIndex::forward(RWTxn& txn) { Stage::Result ret{Stage::Result::kSuccess}; @@ -67,7 +72,7 @@ Stage::Result HistoryIndex::forward(RWTxn& txn) { previous_progress_storage = prune_mode_history_.value_from_head(target_progress); } - collector_ = std::make_unique(etl_settings_); + collector_ = std::make_unique(etl_settings_); if (previous_progress_accounts < target_progress) { success_or_throw(forward_impl(txn, previous_progress_accounts, target_progress, false)); txn.commit_and_renew(); diff --git a/silkworm/node/stagedsync/stages/stage_history_index.hpp b/silkworm/node/stagedsync/stages/stage_history_index.hpp index 5e88ef621a..87ee5f2114 100644 --- a/silkworm/node/stagedsync/stages/stage_history_index.hpp +++ b/silkworm/node/stagedsync/stages/stage_history_index.hpp @@ -28,7 +28,7 @@ class HistoryIndex : public Stage { HistoryIndex( SyncContext* sync_context, size_t batch_size, - db::etl::CollectorSettings etl_settings, + etl::CollectorSettings etl_settings, db::BlockAmount prune_mode_history) : Stage(sync_context, db::stages::kHistoryIndexKey), batch_size_(batch_size), @@ -45,11 +45,11 @@ class HistoryIndex : public Stage { private: size_t batch_size_; - db::etl::CollectorSettings etl_settings_; + etl::CollectorSettings etl_settings_; db::BlockAmount prune_mode_history_; - std::unique_ptr collector_; - std::unique_ptr index_loader_; + std::unique_ptr collector_; + std::unique_ptr index_loader_; std::atomic_bool loading_{false}; // Whether we're in ETL loading phase std::string current_source_; // Current source of data diff --git a/silkworm/node/stagedsync/stages/stage_history_index_test.cpp b/silkworm/node/stagedsync/stages/stage_history_index_test.cpp index 1125af192e..1a5df62a85 100644 --- a/silkworm/node/stagedsync/stages/stage_history_index_test.cpp +++ b/silkworm/node/stagedsync/stages/stage_history_index_test.cpp @@ -34,6 +34,7 @@ namespace silkworm { using namespace evmc::literals; using namespace silkworm::db; +using namespace silkworm::sw_mdbx; using db::test_util::TempChainData; stagedsync::HistoryIndex make_stage_history_index( diff --git a/silkworm/node/stagedsync/stages/stage_interhashes.cpp b/silkworm/node/stagedsync/stages/stage_interhashes.cpp index 4e11c8f2bc..8193546400 100644 --- a/silkworm/node/stagedsync/stages/stage_interhashes.cpp +++ b/silkworm/node/stagedsync/stages/stage_interhashes.cpp @@ -34,7 +34,9 @@ namespace silkworm::stagedsync { using namespace silkworm::db; -using etl_mdbx::Collector; +using sw_mdbx::Collector; +using sw_mdbx::from_slice; +using sw_mdbx::to_slice; Stage::Result InterHashes::forward(RWTxn& txn) { Stage::Result ret{Stage::Result::kSuccess}; @@ -268,7 +270,7 @@ trie::PrefixSet InterHashes::collect_account_changes(RWTxn& txn, BlockNum from, if (auto item{plainstate_accounts.get(address)}; item != nullptr) { plainstate_account = *item; } else { - auto ps_data{plain_state->find(to_slice(address), false)}; + auto ps_data{plain_state->find(db::to_slice(address), false)}; if (ps_data && !ps_data.value.empty()) { const auto account{Account::from_encoded_storage(from_slice(ps_data.value))}; success_or_throw(account); diff --git a/silkworm/node/stagedsync/stages/stage_interhashes.hpp b/silkworm/node/stagedsync/stages/stage_interhashes.hpp index af3233fd37..e34ada71bb 100644 --- a/silkworm/node/stagedsync/stages/stage_interhashes.hpp +++ b/silkworm/node/stagedsync/stages/stage_interhashes.hpp @@ -31,7 +31,7 @@ class InterHashes final : public Stage { InterHashes( SyncContext* sync_context, db::DataModelFactory data_model_factory, - db::etl::CollectorSettings etl_settings) + etl::CollectorSettings etl_settings) : Stage(sync_context, db::stages::kIntermediateHashesKey), data_model_factory_(std::move(data_model_factory)), etl_settings_(std::move(etl_settings)) {} @@ -119,11 +119,11 @@ class InterHashes final : public Stage { db::DataModelFactory data_model_factory_; - db::etl::CollectorSettings etl_settings_; + etl::CollectorSettings etl_settings_; - std::unique_ptr account_collector_; // To accumulate new records for kTrieOfAccounts - std::unique_ptr storage_collector_; // To accumulate new records for kTrieOfStorage - std::unique_ptr loading_collector_; // Effectively the current collector undergoing load (for log) + std::unique_ptr account_collector_; // To accumulate new records for kTrieOfAccounts + std::unique_ptr storage_collector_; // To accumulate new records for kTrieOfStorage + std::unique_ptr loading_collector_; // Effectively the current collector undergoing load (for log) // Logger info std::mutex log_mtx_{}; // Guards async logging diff --git a/silkworm/node/stagedsync/stages/stage_interhashes/_test.cpp b/silkworm/node/stagedsync/stages/stage_interhashes/_test.cpp index cab22c8327..82f4190a9c 100644 --- a/silkworm/node/stagedsync/stages/stage_interhashes/_test.cpp +++ b/silkworm/node/stagedsync/stages/stage_interhashes/_test.cpp @@ -31,7 +31,8 @@ namespace silkworm::trie { using namespace silkworm::db; -using etl_mdbx::Collector; +using namespace silkworm::sw_mdbx; +using sw_mdbx::Collector; static ethash::hash256 keccak256(const evmc::address& address) { return silkworm::keccak256(address.bytes); diff --git a/silkworm/node/stagedsync/stages/stage_interhashes/trie_cursor.cpp b/silkworm/node/stagedsync/stages/stage_interhashes/trie_cursor.cpp index 057d8041ae..08d10c24d7 100644 --- a/silkworm/node/stagedsync/stages/stage_interhashes/trie_cursor.cpp +++ b/silkworm/node/stagedsync/stages/stage_interhashes/trie_cursor.cpp @@ -73,7 +73,7 @@ void SubNode::parse(ByteView k, ByteView v) { deleted = false; } -TrieCursor::TrieCursor(db::ROCursor& db_cursor, PrefixSet* changed, db::etl::Collector* collector) +TrieCursor::TrieCursor(sw_mdbx::ROCursor& db_cursor, PrefixSet* changed, etl::Collector* collector) : db_cursor_(db_cursor), changed_list_{changed}, collector_{collector} { curr_key_.reserve(64); prev_key_.reserve(64); @@ -223,21 +223,21 @@ TrieCursor::MoveOperationResult TrieCursor::to_next() { bool TrieCursor::db_seek(ByteView seek_key) { buffer_.assign(prefix_).append(seek_key); - const auto buffer_slice{db::to_slice(buffer_)}; + const auto buffer_slice = sw_mdbx::to_slice(buffer_); auto data{buffer_.empty() ? db_cursor_.to_first(false) : db_cursor_.lower_bound(buffer_slice, false)}; if (!data || !data.key.starts_with(buffer_slice)) { return false; } - ByteView db_cursor_key{db::from_slice(data.key)}; // Save db_cursor_ key ... - db_cursor_key.remove_prefix(prefix_.length()); // ... and remove prefix_ so we have node key + ByteView db_cursor_key = sw_mdbx::from_slice(data.key); // Save db_cursor_ key ... + db_cursor_key.remove_prefix(prefix_.length()); // ... and remove prefix_ so we have node key if (seek_key.empty() && !db_cursor_key.empty()) { // Note ! an empty seek_key means we're looking for a root node with empty key which does not exist return false; } - ByteView db_cursor_val{db::from_slice(data.value)}; // Save db_cursor_ value - level_ += seek_key.empty() ? 0 : 1u; // Down one level for child node. Stay at zero for root node + ByteView db_cursor_val = sw_mdbx::from_slice(data.value); // Save db_cursor_ value + level_ += seek_key.empty() ? 0 : 1u; // Down one level for child node. Stay at zero for root node auto& new_node{sub_nodes_[level_]}; new_node.parse(db_cursor_key, db_cursor_val); return true; diff --git a/silkworm/node/stagedsync/stages/stage_interhashes/trie_cursor.hpp b/silkworm/node/stagedsync/stages/stage_interhashes/trie_cursor.hpp index e9f9826ac0..d44e1e6088 100644 --- a/silkworm/node/stagedsync/stages/stage_interhashes/trie_cursor.hpp +++ b/silkworm/node/stagedsync/stages/stage_interhashes/trie_cursor.hpp @@ -73,7 +73,7 @@ class SubNode : public Node { class TrieCursor { public: - explicit TrieCursor(db::ROCursor& db_cursor, PrefixSet* changed, db::etl::Collector* collector = nullptr); + explicit TrieCursor(sw_mdbx::ROCursor& db_cursor, PrefixSet* changed, etl::Collector* collector = nullptr); // Not copyable nor movable TrieCursor(const TrieCursor&) = delete; @@ -104,10 +104,10 @@ class TrieCursor { Bytes prefix_{}; // Db key prefix for this trie (0 bytes TrieAccount - 40 bytes TrieStorage) Bytes buffer_{}; // A convenience buffer - db::ROCursor& db_cursor_; // The underlying db cursor (TrieAccount/TrieStorage) - PrefixSet* changed_list_; // The collection of changed nibbled keys - ByteView next_created_{}; // The next created account/location in changed list - db::etl::Collector* collector_; // Pointer to a collector for deletion of obsolete keys + sw_mdbx::ROCursor& db_cursor_; // The underlying db cursor (TrieAccount/TrieStorage) + PrefixSet* changed_list_; // The collection of changed nibbled keys + ByteView next_created_{}; // The next created account/location in changed list + etl::Collector* collector_; // Pointer to a collector for deletion of obsolete keys bool db_seek(ByteView seek_key); // Seeks lowerbound of provided key using db_cursor_ void db_delete(SubNode& node); // Collects deletion of node being rebuilt or no longer needed diff --git a/silkworm/node/stagedsync/stages/stage_interhashes/trie_loader.cpp b/silkworm/node/stagedsync/stages/stage_interhashes/trie_loader.cpp index f562f3bbf6..2205d33c5b 100644 --- a/silkworm/node/stagedsync/stages/stage_interhashes/trie_loader.cpp +++ b/silkworm/node/stagedsync/stages/stage_interhashes/trie_loader.cpp @@ -28,6 +28,7 @@ namespace silkworm::trie { using namespace silkworm::db; +using namespace silkworm::sw_mdbx; TrieLoader::TrieLoader( ROTxn& txn, diff --git a/silkworm/node/stagedsync/stages/stage_interhashes/trie_loader.hpp b/silkworm/node/stagedsync/stages/stage_interhashes/trie_loader.hpp index 8c35d413a0..80f86cd15b 100644 --- a/silkworm/node/stagedsync/stages/stage_interhashes/trie_loader.hpp +++ b/silkworm/node/stagedsync/stages/stage_interhashes/trie_loader.hpp @@ -27,11 +27,11 @@ namespace silkworm::trie { class TrieLoader { public: explicit TrieLoader( - db::ROTxn& txn, + sw_mdbx::ROTxn& txn, PrefixSet* account_changes, PrefixSet* storage_changes, - db::etl::Collector* account_trie_node_collector, - db::etl::Collector* storage_trie_node_collector); + etl::Collector* account_trie_node_collector, + etl::Collector* storage_trie_node_collector); //! \brief (re)calculates root hash on behalf of collected hashed changes and existing data in TrieOfAccount and //! TrieOfStorage buckets @@ -46,11 +46,11 @@ class TrieLoader { } private: - db::ROTxn& txn_; + sw_mdbx::ROTxn& txn_; PrefixSet* account_changes_; PrefixSet* storage_changes_; - db::etl::Collector* account_trie_node_collector_; - db::etl::Collector* storage_trie_node_collector_; + etl::Collector* account_trie_node_collector_; + etl::Collector* storage_trie_node_collector_; std::string log_key_{}; // To export logging key mutable std::mutex log_mtx_{}; // Guards async logging @@ -62,7 +62,7 @@ class TrieLoader { static evmc::bytes32 calculate_storage_root( TrieCursor& trie_storage_cursor, HashBuilder& storage_hash_builder, - db::ROCursorDupSort& hashed_storage, + sw_mdbx::ROCursorDupSort& hashed_storage, const Bytes& db_storage_prefix); }; } // namespace silkworm::trie diff --git a/silkworm/node/stagedsync/stages/stage_log_index.cpp b/silkworm/node/stagedsync/stages/stage_log_index.cpp index 3c67162db7..b1d40994c6 100644 --- a/silkworm/node/stagedsync/stages/stage_log_index.cpp +++ b/silkworm/node/stagedsync/stages/stage_log_index.cpp @@ -26,6 +26,10 @@ namespace silkworm::stagedsync { using namespace silkworm::db; +using silkworm::sw_mdbx::to_slice; +namespace bitmap { + using namespace silkworm::sw_mdbx::bitmap; +} namespace { //! LogBitmapBuilder is a CBOR consumer which builds address and topic roaring bitmaps from the CBOR @@ -256,7 +260,7 @@ Stage::Result LogIndex::prune(RWTxn& txn) { } void LogIndex::forward_impl(RWTxn& txn, const BlockNum from, const BlockNum to) { - using etl_mdbx::Collector; + using sw_mdbx::Collector; const MapConfig source_config{table::kLogs}; diff --git a/silkworm/node/stagedsync/stages/stage_log_index.hpp b/silkworm/node/stagedsync/stages/stage_log_index.hpp index bdae9bffc9..972185194f 100644 --- a/silkworm/node/stagedsync/stages/stage_log_index.hpp +++ b/silkworm/node/stagedsync/stages/stage_log_index.hpp @@ -30,7 +30,7 @@ class LogIndex : public Stage { LogIndex( SyncContext* sync_context, size_t batch_size, - db::etl::CollectorSettings etl_settings, + etl::CollectorSettings etl_settings, db::BlockAmount prune_mode_history) : Stage(sync_context, db::stages::kLogIndexKey), batch_size_(batch_size), @@ -47,12 +47,12 @@ class LogIndex : public Stage { private: size_t batch_size_; - db::etl::CollectorSettings etl_settings_; + etl::CollectorSettings etl_settings_; db::BlockAmount prune_mode_history_; - std::unique_ptr topics_collector_; - std::unique_ptr addresses_collector_; - std::unique_ptr index_loader_; + std::unique_ptr topics_collector_; + std::unique_ptr addresses_collector_; + std::unique_ptr index_loader_; std::atomic_bool loading_{false}; // Whether we're in ETL loading phase std::string current_source_; // Current source of data diff --git a/silkworm/node/stagedsync/stages/stage_senders.cpp b/silkworm/node/stagedsync/stages/stage_senders.cpp index 3236bf4d76..0c3abbdcc5 100644 --- a/silkworm/node/stagedsync/stages/stage_senders.cpp +++ b/silkworm/node/stagedsync/stages/stage_senders.cpp @@ -33,6 +33,8 @@ namespace silkworm::stagedsync { using namespace std::chrono_literals; using namespace silkworm::db; +using silkworm::sw_mdbx::from_slice; +using silkworm::sw_mdbx::to_slice; Senders::Senders( SyncContext* sync_context, @@ -59,7 +61,7 @@ Stage::Result Senders::forward(RWTxn& txn) { total_collected_transactions_ = 0; log_lock.unlock(); - collector_ = std::make_unique(etl_settings_); + collector_ = std::make_unique(etl_settings_); const auto res{parallel_recover(txn)}; if (res == Stage::Result::kSuccess) { diff --git a/silkworm/node/stagedsync/stages/stage_senders.hpp b/silkworm/node/stagedsync/stages/stage_senders.hpp index 433048331e..c87f83d37c 100644 --- a/silkworm/node/stagedsync/stages/stage_senders.hpp +++ b/silkworm/node/stagedsync/stages/stage_senders.hpp @@ -56,7 +56,7 @@ class Senders final : public Stage { db::DataModelFactory data_model_factory, const ChainConfig& chain_config, size_t batch_size, - db::etl::CollectorSettings etl_settings, + etl::CollectorSettings etl_settings, db::BlockAmount prune_mode_senders); ~Senders() override = default; @@ -96,8 +96,8 @@ class Senders final : public Stage { uint64_t collected_senders_{0}; //! ETL collector writing recovered senders in bulk - db::etl::CollectorSettings etl_settings_; - std::unique_ptr collector_; + etl::CollectorSettings etl_settings_; + std::unique_ptr collector_; // Stats std::mutex mutex_{}; diff --git a/silkworm/node/stagedsync/stages/stage_triggers.hpp b/silkworm/node/stagedsync/stages/stage_triggers.hpp index e66340ca0e..b16e0de3b8 100644 --- a/silkworm/node/stagedsync/stages/stage_triggers.hpp +++ b/silkworm/node/stagedsync/stages/stage_triggers.hpp @@ -23,7 +23,7 @@ namespace silkworm::stagedsync { -class TriggersStage : public Stage, public StageScheduler { +class TriggersStage : public Stage, public datastore::StageScheduler { public: explicit TriggersStage(SyncContext* sync_context); ~TriggersStage() override = default; diff --git a/silkworm/node/stagedsync/stages/stage_tx_lookup.cpp b/silkworm/node/stagedsync/stages/stage_tx_lookup.cpp index fe7a2094b0..d06567feef 100644 --- a/silkworm/node/stagedsync/stages/stage_tx_lookup.cpp +++ b/silkworm/node/stagedsync/stages/stage_tx_lookup.cpp @@ -27,7 +27,7 @@ namespace silkworm::stagedsync { using namespace silkworm::db; -using etl_mdbx::Collector; +using sw_mdbx::Collector; Stage::Result TxLookup::forward(RWTxn& txn) { Stage::Result ret{Stage::Result::kSuccess}; diff --git a/silkworm/node/stagedsync/stages/stage_tx_lookup.hpp b/silkworm/node/stagedsync/stages/stage_tx_lookup.hpp index 476f6cb579..2d41c95954 100644 --- a/silkworm/node/stagedsync/stages/stage_tx_lookup.hpp +++ b/silkworm/node/stagedsync/stages/stage_tx_lookup.hpp @@ -29,7 +29,7 @@ class TxLookup : public Stage { TxLookup( SyncContext* sync_context, db::DataModelFactory data_model_factory, - db::etl::CollectorSettings etl_settings, + etl::CollectorSettings etl_settings, db::BlockAmount prune_mode_tx_index) : Stage(sync_context, db::stages::kTxLookupKey), data_model_factory_(std::move(data_model_factory)), @@ -46,10 +46,10 @@ class TxLookup : public Stage { private: db::DataModelFactory data_model_factory_; - db::etl::CollectorSettings etl_settings_; + etl::CollectorSettings etl_settings_; db::BlockAmount prune_mode_tx_index_; - std::unique_ptr collector_; + std::unique_ptr collector_; std::atomic_bool loading_{false}; // Whether we're in ETL loading phase std::string current_source_; // Current source of data diff --git a/silkworm/node/stagedsync/stages/stage_tx_lookup_test.cpp b/silkworm/node/stagedsync/stages/stage_tx_lookup_test.cpp index a7662923a7..9c284ed4a9 100644 --- a/silkworm/node/stagedsync/stages/stage_tx_lookup_test.cpp +++ b/silkworm/node/stagedsync/stages/stage_tx_lookup_test.cpp @@ -26,6 +26,7 @@ namespace silkworm { using namespace silkworm::db; +using namespace silkworm::sw_mdbx; using namespace evmc::literals; using db::test_util::TempChainDataStore; diff --git a/silkworm/node/stagedsync/stages_test.cpp b/silkworm/node/stagedsync/stages_test.cpp index 61ffcedcab..7899482441 100644 --- a/silkworm/node/stagedsync/stages_test.cpp +++ b/silkworm/node/stagedsync/stages_test.cpp @@ -39,6 +39,7 @@ using namespace silkworm; using namespace silkworm::db; +using namespace silkworm::sw_mdbx; using namespace evmc::literals; static ethash::hash256 keccak256(const evmc::address& address) { @@ -90,7 +91,7 @@ TEST_CASE("Sync Stages") { node_settings.data_directory->snapshots().path(), }; - RWTxnManaged txn = data_store.chaindata_rw().start_rw_tx(); + auto txn = data_store.chaindata_rw().start_rw_tx(); table::check_or_create_chaindata_tables(txn); txn.commit_and_renew(); const auto initial_tx_sequence{read_map_sequence(txn, table::kBlockTransactions.name)}; diff --git a/silkworm/node/test_util/mock_execution_engine.hpp b/silkworm/node/test_util/mock_execution_engine.hpp index 0fc5a7942b..6cd0f355de 100644 --- a/silkworm/node/test_util/mock_execution_engine.hpp +++ b/silkworm/node/test_util/mock_execution_engine.hpp @@ -41,7 +41,7 @@ class MockExecutionEngine : public stagedsync::ExecutionEngine { }; }; - MockExecutionEngine(boost::asio::any_io_executor executor, NodeSettings& ns, db::RWAccess dba) + MockExecutionEngine(boost::asio::any_io_executor executor, NodeSettings& ns, sw_mdbx::RWAccess dba) : ExecutionEngine{ std::move(executor), ns, diff --git a/silkworm/rpc/commands/ots_api.cpp b/silkworm/rpc/commands/ots_api.cpp index 844532662e..9e5dcd9799 100644 --- a/silkworm/rpc/commands/ots_api.cpp +++ b/silkworm/rpc/commands/ots_api.cpp @@ -42,6 +42,9 @@ namespace silkworm::rpc::commands { using namespace silkworm::db; using db::kv::StateReader; +namespace bitmap { + using namespace silkworm::sw_mdbx::bitmap; +} static constexpr int kCurrentApiLevel{8}; diff --git a/silkworm/rpc/daemon.cpp b/silkworm/rpc/daemon.cpp index 0428e58f0c..cb8de68aa0 100644 --- a/silkworm/rpc/daemon.cpp +++ b/silkworm/rpc/daemon.cpp @@ -126,7 +126,7 @@ int Daemon::run(const DaemonSettings& settings) { if (settings.datadir) { DataDirectory data_dir{*settings.datadir}; - silkworm::db::EnvConfig db_config{ + silkworm::sw_mdbx::EnvConfig db_config{ .path = data_dir.chaindata().path().string(), .in_memory = true, .shared = true, @@ -138,7 +138,7 @@ int Daemon::run(const DaemonSettings& settings) { }); // At startup check that chain configuration is valid - db::ROTxnManaged ro_txn = data_store->chaindata().start_ro_tx(); + sw_mdbx::ROTxnManaged ro_txn = data_store->chaindata().start_ro_tx(); db::DataModel data_access = db::DataModelFactory{data_store->ref()}(ro_txn); if (const auto chain_config{data_access.read_chain_config()}; !chain_config) { throw std::runtime_error{"invalid chain configuration"}; @@ -353,7 +353,7 @@ void Daemon::start() { // Put the interface logs into the data folder std::filesystem::path data_folder{}; if (data_store_) { - db::RWAccess& chaindata = data_store_->chaindata; + sw_mdbx::RWAccess& chaindata = data_store_->chaindata; mdbx::env& chaindata_env = *chaindata; auto chaindata_path = chaindata_env.get_path(); // Trick to remove any empty filename because MDBX chaindata path ends with '/' diff --git a/silkworm/sync/internals/body_sequence_test.cpp b/silkworm/sync/internals/body_sequence_test.cpp index d6ce4946e6..3964719101 100644 --- a/silkworm/sync/internals/body_sequence_test.cpp +++ b/silkworm/sync/internals/body_sequence_test.cpp @@ -448,7 +448,7 @@ TEST_CASE("body downloading", "[silkworm][sync][BodySequence]") { BlockHeader header2; header2.number = 2; header2.parent_hash = header1_hash; - db::RWTxnManaged txn2 = context.chaindata_rw().start_rw_tx(); + sw_mdbx::RWTxnManaged txn2 = context.chaindata_rw().start_rw_tx(); db::write_canonical_header_hash(txn2, header2.hash().bytes, 1); db::write_canonical_header(txn2, header2); db::write_header(txn2, header2, true); diff --git a/silkworm/sync/internals/header_retrieval_test.cpp b/silkworm/sync/internals/header_retrieval_test.cpp index 0c4d9921f0..7431b22dc1 100644 --- a/silkworm/sync/internals/header_retrieval_test.cpp +++ b/silkworm/sync/internals/header_retrieval_test.cpp @@ -28,7 +28,7 @@ TEST_CASE("HeaderRetrieval") { context.add_genesis_data(); context.commit_txn(); - db::ROTxnManaged tx = context->chaindata().start_ro_tx(); + sw_mdbx::ROTxnManaged tx = context->chaindata().start_ro_tx(); db::DataModel data_model = context.data_model_factory()(tx); HeaderRetrieval header_retrieval{data_model}; diff --git a/silkworm/sync/messages/inbound_get_block_bodies.cpp b/silkworm/sync/messages/inbound_get_block_bodies.cpp index a80747c983..da071d8207 100644 --- a/silkworm/sync/messages/inbound_get_block_bodies.cpp +++ b/silkworm/sync/messages/inbound_get_block_bodies.cpp @@ -40,7 +40,7 @@ void InboundGetBlockBodies::execute(db::DataStoreRef db, HeaderChain&, BodySeque if (bs.max_block_in_output() == 0) return; - db::ROTxnManaged tx = db.chaindata.start_ro_tx(); + sw_mdbx::ROTxnManaged tx = db.chaindata.start_ro_tx(); BodyRetrieval body_retrieval{tx}; BlockBodiesPacket66 reply; diff --git a/silkworm/sync/messages/inbound_get_block_headers.cpp b/silkworm/sync/messages/inbound_get_block_headers.cpp index f3c544460b..d7f9f49965 100644 --- a/silkworm/sync/messages/inbound_get_block_headers.cpp +++ b/silkworm/sync/messages/inbound_get_block_headers.cpp @@ -40,7 +40,7 @@ void InboundGetBlockHeaders::execute(db::DataStoreRef db, HeaderChain&, BodySequ if (bs.max_block_in_output() == 0) // skip requests in the first sync even if we already saved some headers return; - db::ROTxnManaged tx = db.chaindata.start_ro_tx(); + sw_mdbx::ROTxnManaged tx = db.chaindata.start_ro_tx(); db::DataModel data_model{tx, db.blocks_repository}; HeaderRetrieval header_retrieval(data_model);