diff --git a/nano/core_test/CMakeLists.txt b/nano/core_test/CMakeLists.txt index ba92b42681..23c2274428 100644 --- a/nano/core_test/CMakeLists.txt +++ b/nano/core_test/CMakeLists.txt @@ -38,6 +38,7 @@ add_executable( processor_service.cpp rep_crawler.cpp peer_container.cpp + rep_weight_store.cpp scheduler_buckets.cpp request_aggregator.cpp signal_manager.cpp diff --git a/nano/core_test/rep_weight_store.cpp b/nano/core_test/rep_weight_store.cpp new file mode 100644 index 0000000000..90d8db2c29 --- /dev/null +++ b/nano/core_test/rep_weight_store.cpp @@ -0,0 +1,75 @@ +#include +#include +#include +#include + +#include + +#include +#include + +TEST (rep_weight_store, empty) +{ + auto store = nano::test::make_store (); + ASSERT_TRUE (!store->init_error ()); + auto txn{ store->tx_begin_read () }; + ASSERT_EQ (0, store->rep_weight.count (txn)); +} + +TEST (rep_weight_store, add_item) +{ + auto store = nano::test::make_store (); + ASSERT_TRUE (!store->init_error ()); + auto txn{ store->tx_begin_write () }; + + nano::account representative{ 123 }; + nano::uint128_t weight{ 456 }; + store->rep_weight.put (txn, representative, weight); + + ASSERT_EQ (1, store->rep_weight.count (txn)); + ASSERT_EQ (weight, store->rep_weight.get (txn, representative)); +} + +TEST (rep_weight_store, del) +{ + auto store = nano::test::make_store (); + ASSERT_TRUE (!store->init_error ()); + auto txn{ store->tx_begin_write () }; + + store->rep_weight.put (txn, 1, 100); + store->rep_weight.put (txn, 2, 200); + store->rep_weight.put (txn, 3, 300); + + store->rep_weight.del (txn, 2); + + ASSERT_EQ (2, store->rep_weight.count (txn)); + ASSERT_EQ (0, store->rep_weight.get (txn, 200)); +} + +TEST (rep_weight_store, for_each_par) +{ + auto store = nano::test::make_store (); + ASSERT_TRUE (!store->init_error ()); + { + auto txn{ store->tx_begin_write () }; + for (auto i = 0; i < 50; ++i) + { + store->rep_weight.put (txn, i, 100); + } + } + + std::atomic_size_t rep_total{ 0 }; + std::atomic_size_t weight_total{ 0 }; + + store->rep_weight.for_each_par ( + [&rep_total, &weight_total] (auto const &, auto i, auto n) { + for (; i != n; ++i) + { + rep_total.fetch_add (static_cast (i->first.number ())); + weight_total.fetch_add (static_cast (i->second.number ())); + } + }); + + ASSERT_EQ (1225, rep_total.load ()); + ASSERT_EQ (50 * 100, weight_total.load ()); +} diff --git a/nano/node/make_store.cpp b/nano/node/make_store.cpp index c3c8c28105..87c15700fe 100644 --- a/nano/node/make_store.cpp +++ b/nano/node/make_store.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/nano/store/CMakeLists.txt b/nano/store/CMakeLists.txt index bf79b03e34..9ea6c5c739 100644 --- a/nano/store/CMakeLists.txt +++ b/nano/store/CMakeLists.txt @@ -25,6 +25,7 @@ add_library( lmdb/peer.hpp lmdb/pending.hpp lmdb/pruned.hpp + lmdb/rep_weight.hpp lmdb/transaction_impl.hpp lmdb/version.hpp lmdb/wallet_value.hpp @@ -43,6 +44,7 @@ add_library( rocksdb/peer.hpp rocksdb/pending.hpp rocksdb/pruned.hpp + rocksdb/rep_weight.hpp rocksdb/rocksdb.hpp rocksdb/iterator.hpp rocksdb/transaction_impl.hpp @@ -73,6 +75,7 @@ add_library( lmdb/peer.cpp lmdb/pending.cpp lmdb/pruned.cpp + lmdb/rep_weight.cpp lmdb/version.cpp lmdb/wallet_value.cpp online_weight.cpp @@ -89,6 +92,7 @@ add_library( rocksdb/peer.cpp rocksdb/pending.cpp rocksdb/pruned.cpp + rocksdb/rep_weight.cpp rocksdb/rocksdb.cpp rocksdb/transaction.cpp rocksdb/version.cpp diff --git a/nano/store/component.cpp b/nano/store/component.cpp index 838a7b9cb7..9e4b7e9dbc 100644 --- a/nano/store/component.cpp +++ b/nano/store/component.cpp @@ -6,8 +6,9 @@ #include #include #include +#include -nano::store::component::component (nano::store::block & block_store_a, nano::store::frontier & frontier_store_a, nano::store::account & account_store_a, nano::store::pending & pending_store_a, nano::store::online_weight & online_weight_store_a, nano::store::pruned & pruned_store_a, nano::store::peer & peer_store_a, nano::store::confirmation_height & confirmation_height_store_a, nano::store::final_vote & final_vote_store_a, nano::store::version & version_store_a) : +nano::store::component::component (nano::store::block & block_store_a, nano::store::frontier & frontier_store_a, nano::store::account & account_store_a, nano::store::pending & pending_store_a, nano::store::online_weight & online_weight_store_a, nano::store::pruned & pruned_store_a, nano::store::peer & peer_store_a, nano::store::confirmation_height & confirmation_height_store_a, nano::store::final_vote & final_vote_store_a, nano::store::version & version_store_a, nano::store::rep_weight & rep_weight_a) : block (block_store_a), frontier (frontier_store_a), account (account_store_a), @@ -17,7 +18,8 @@ nano::store::component::component (nano::store::block & block_store_a, nano::sto peer (peer_store_a), confirmation_height (confirmation_height_store_a), final_vote (final_vote_store_a), - version (version_store_a) + version (version_store_a), + rep_weight (rep_weight_a) { } diff --git a/nano/store/component.hpp b/nano/store/component.hpp index a675ca5623..eaa8faf958 100644 --- a/nano/store/component.hpp +++ b/nano/store/component.hpp @@ -27,6 +27,7 @@ namespace store class pending; class pruned; class version; + class rep_weight; } class ledger_cache; @@ -52,7 +53,8 @@ namespace store nano::store::peer &, nano::store::confirmation_height &, nano::store::final_vote &, - nano::store::version & + nano::store::version &, + nano::store::rep_weight & ); // clang-format on virtual ~component () = default; @@ -68,6 +70,7 @@ namespace store store::frontier & frontier; store::account & account; store::pending & pending; + store::rep_weight & rep_weight; static int constexpr version_minimum{ 21 }; static int constexpr version_current{ 22 }; diff --git a/nano/store/lmdb/lmdb.cpp b/nano/store/lmdb/lmdb.cpp index 4dd9e64407..aa5ea75957 100644 --- a/nano/store/lmdb/lmdb.cpp +++ b/nano/store/lmdb/lmdb.cpp @@ -24,7 +24,8 @@ nano::store::lmdb::component::component (nano::logger & logger_a, std::filesyste peer_store, confirmation_height_store, final_vote_store, - version_store + version_store, + rep_weight_store }, // clang-format on block_store{ *this }, @@ -37,6 +38,7 @@ nano::store::lmdb::component::component (nano::logger & logger_a, std::filesyste confirmation_height_store{ *this }, final_vote_store{ *this }, version_store{ *this }, + rep_weight_store{ *this }, logger{ logger_a }, env (error, path_a, nano::store::lmdb::env::options::make ().set_config (lmdb_config_a).set_use_no_mem_init (true)), mdb_txn_tracker (logger_a, txn_tracking_config_a, block_processor_batch_max_time_a), @@ -204,6 +206,7 @@ void nano::store::lmdb::component::open_databases (bool & error_a, store::transa pending_store.pending_handle = pending_store.pending_v0_handle; error_a |= mdb_dbi_open (env.tx (transaction_a), "final_votes", flags, &final_vote_store.final_votes_handle) != 0; error_a |= mdb_dbi_open (env.tx (transaction_a), "blocks", MDB_CREATE, &block_store.blocks_handle) != 0; + error_a |= mdb_dbi_open (env.tx (transaction_a), "rep_weights", flags, &rep_weight_store.rep_weights_handle) != 0; } bool nano::store::lmdb::component::do_upgrades (store::write_transaction & transaction_a, nano::ledger_constants & constants, bool & needs_vacuuming) @@ -339,6 +342,8 @@ MDB_dbi nano::store::lmdb::component::table_to_dbi (tables table_a) const return confirmation_height_store.confirmation_height_handle; case tables::final_votes: return final_vote_store.final_votes_handle; + case tables::rep_weights: + return rep_weight_store.rep_weights_handle; default: release_assert (false); return peer_store.peers_handle; diff --git a/nano/store/lmdb/lmdb.hpp b/nano/store/lmdb/lmdb.hpp index a1f5dda6cf..90bbc8ca6c 100644 --- a/nano/store/lmdb/lmdb.hpp +++ b/nano/store/lmdb/lmdb.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -50,6 +51,7 @@ class component : public nano::store::component nano::store::lmdb::pending pending_store; nano::store::lmdb::pruned pruned_store; nano::store::lmdb::version version_store; + nano::store::lmdb::rep_weight rep_weight_store; friend class nano::store::lmdb::account; friend class nano::store::lmdb::block; @@ -61,6 +63,7 @@ class component : public nano::store::component friend class nano::store::lmdb::pending; friend class nano::store::lmdb::pruned; friend class nano::store::lmdb::version; + friend class nano::store::lmdb::rep_weight; public: component (nano::logger &, std::filesystem::path const &, nano::ledger_constants & constants, nano::txn_tracking_config const & txn_tracking_config_a = nano::txn_tracking_config{}, std::chrono::milliseconds block_processor_batch_max_time_a = std::chrono::milliseconds (5000), nano::lmdb_config const & lmdb_config_a = nano::lmdb_config{}, bool backup_before_upgrade = false); diff --git a/nano/store/lmdb/rep_weight.cpp b/nano/store/lmdb/rep_weight.cpp new file mode 100644 index 0000000000..529efae68e --- /dev/null +++ b/nano/store/lmdb/rep_weight.cpp @@ -0,0 +1,68 @@ +#include +#include +#include +#include + +#include +#include + +nano::store::lmdb::rep_weight::rep_weight (nano::store::lmdb::component & store_a) : + store{ store_a } +{ +} + +uint64_t nano::store::lmdb::rep_weight::count (store::transaction const & txn_a) +{ + return store.count (txn_a, tables::rep_weights); +} + +nano::uint128_t nano::store::lmdb::rep_weight::get (store::transaction const & txn_a, nano::account const & representative_a) +{ + nano::store::lmdb::db_val value; + auto status = store.get (txn_a, tables::rep_weights, representative_a, value); + release_assert (store.success (status) || store.not_found (status)); + nano::uint128_t weight{ 0 }; + if (store.success (status)) + { + nano::uint128_union weight_union{ value }; + weight = weight_union.number (); + } + return weight; +} + +void nano::store::lmdb::rep_weight::put (store::write_transaction const & txn_a, nano::account const & representative_a, nano::uint128_t const & weight_a) +{ + nano::uint128_union weight{ weight_a }; + auto status = store.put (txn_a, tables::rep_weights, representative_a, weight); + store.release_assert_success (status); +} + +void nano::store::lmdb::rep_weight::del (store::write_transaction const & txn_a, nano::account const & representative_a) +{ + auto status = store.del (txn_a, tables::rep_weights, representative_a); + store.release_assert_success (status); +} + +nano::store::iterator nano::store::lmdb::rep_weight::begin (store::transaction const & transaction_a, nano::account const & representative_a) const +{ + return store.make_iterator (transaction_a, tables::rep_weights, representative_a); +} + +nano::store::iterator nano::store::lmdb::rep_weight::begin (store::transaction const & transaction_a) const +{ + return store.make_iterator (transaction_a, tables::rep_weights); +} + +nano::store::iterator nano::store::lmdb::rep_weight::end () const +{ + return nano::store::iterator (nullptr); +} + +void nano::store::lmdb::rep_weight::for_each_par (std::function, store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} diff --git a/nano/store/lmdb/rep_weight.hpp b/nano/store/lmdb/rep_weight.hpp new file mode 100644 index 0000000000..2a6ef53c5c --- /dev/null +++ b/nano/store/lmdb/rep_weight.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include + +namespace nano::store::lmdb +{ +class component; + +class rep_weight : public nano::store::rep_weight +{ +private: + nano::store::lmdb::component & store; + +public: + explicit rep_weight (nano::store::lmdb::component & store_a); + + uint64_t count (store::transaction const & txn) override; + nano::uint128_t get (store::transaction const & txn_a, nano::account const & representative_a) override; + void put (store::write_transaction const & txn_a, nano::account const & representative_a, nano::uint128_t const & weight_a) override; + void del (store::write_transaction const &, nano::account const & representative_a) override; + store::iterator begin (store::transaction const & transaction_a, nano::account const & representative_a) const override; + store::iterator begin (store::transaction const & transaction_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; + + /** + * Representative weights + * nano::account -> uint128_t + */ + MDB_dbi rep_weights_handle{ 0 }; +}; +} diff --git a/nano/store/rep_weight.hpp b/nano/store/rep_weight.hpp new file mode 100644 index 0000000000..72ac9f84aa --- /dev/null +++ b/nano/store/rep_weight.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace nano +{ +// class account; +} +namespace nano::store +{ +/** + * A lookup table of all representatives and their vote weight + */ +class rep_weight +{ +public: + virtual ~rep_weight (){}; + virtual uint64_t count (store::transaction const & txn_a) = 0; + virtual nano::uint128_t get (store::transaction const & txn_a, nano::account const & representative_a) = 0; + virtual void put (store::write_transaction const & txn_a, nano::account const & representative_a, nano::uint128_t const & weight_a) = 0; + virtual void del (store::write_transaction const &, nano::account const & representative_a) = 0; + virtual store::iterator begin (store::transaction const & transaction_a, nano::account const & representative_a) const = 0; + virtual store::iterator begin (store::transaction const & transaction_a) const = 0; + virtual store::iterator end () const = 0; + virtual void for_each_par (std::function, store::iterator)> const & action_a) const = 0; +}; +} diff --git a/nano/store/rocksdb/rep_weight.cpp b/nano/store/rocksdb/rep_weight.cpp new file mode 100644 index 0000000000..7f29e61e4b --- /dev/null +++ b/nano/store/rocksdb/rep_weight.cpp @@ -0,0 +1,67 @@ +#include +#include +#include +#include + +#include + +nano::store::rocksdb::rep_weight::rep_weight (nano::store::rocksdb::component & store_a) : + store{ store_a } +{ +} + +uint64_t nano::store::rocksdb::rep_weight::count (store::transaction const & txn_a) +{ + return store.count (txn_a, tables::rep_weights); +} + +nano::uint128_t nano::store::rocksdb::rep_weight::get (store::transaction const & txn_a, nano::account const & representative_a) +{ + db_val value; + auto status = store.get (txn_a, tables::rep_weights, representative_a, value); + release_assert (store.success (status) || store.not_found (status)); + nano::uint128_t weight{ 0 }; + if (store.success (status)) + { + nano::uint128_union weight_union{ value }; + weight = weight_union.number (); + } + return weight; +} + +void nano::store::rocksdb::rep_weight::put (store::write_transaction const & txn_a, nano::account const & representative_a, nano::uint128_t const & weight_a) +{ + nano::uint128_union weight{ weight_a }; + auto status = store.put (txn_a, tables::rep_weights, representative_a, weight); + store.release_assert_success (status); +} + +void nano::store::rocksdb::rep_weight::del (store::write_transaction const & txn_a, nano::account const & representative_a) +{ + auto status = store.del (txn_a, tables::rep_weights, representative_a); + store.release_assert_success (status); +} + +nano::store::iterator nano::store::rocksdb::rep_weight::begin (store::transaction const & txn_a, nano::account const & representative_a) const +{ + return store.make_iterator (txn_a, tables::rep_weights, representative_a); +} + +nano::store::iterator nano::store::rocksdb::rep_weight::begin (store::transaction const & txn_a) const +{ + return store.make_iterator (txn_a, tables::rep_weights); +} + +nano::store::iterator nano::store::rocksdb::rep_weight::end () const +{ + return store::iterator (nullptr); +} + +void nano::store::rocksdb::rep_weight::for_each_par (std::function, store::iterator)> const & action_a) const +{ + parallel_traversal ( + [&action_a, this] (nano::uint256_t const & start, nano::uint256_t const & end, bool const is_last) { + auto transaction (this->store.tx_begin_read ()); + action_a (transaction, this->begin (transaction, start), !is_last ? this->begin (transaction, end) : this->end ()); + }); +} \ No newline at end of file diff --git a/nano/store/rocksdb/rep_weight.hpp b/nano/store/rocksdb/rep_weight.hpp new file mode 100644 index 0000000000..c86f27150b --- /dev/null +++ b/nano/store/rocksdb/rep_weight.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include + +namespace nano::store::rocksdb +{ +class component; +} +namespace nano::store::rocksdb +{ +class rep_weight : public nano::store::rep_weight +{ +private: + nano::store::rocksdb::component & store; + +public: + explicit rep_weight (nano::store::rocksdb::component & store_a); + uint64_t count (store::transaction const & txn_a) override; + nano::uint128_t get (store::transaction const & txn_a, nano::account const & representative_a) override; + void put (store::write_transaction const & txn_a, nano::account const & representative_a, nano::uint128_t const & weight_a) override; + void del (store::write_transaction const &, nano::account const & representative_a) override; + store::iterator begin (store::transaction const & txn_a, nano::account const & representative_a) const override; + store::iterator begin (store::transaction const & txn_a) const override; + store::iterator end () const override; + void for_each_par (std::function, store::iterator)> const & action_a) const override; +}; +} diff --git a/nano/store/rocksdb/rocksdb.cpp b/nano/store/rocksdb/rocksdb.cpp index 43b82f2b4c..515a03772d 100644 --- a/nano/store/rocksdb/rocksdb.cpp +++ b/nano/store/rocksdb/rocksdb.cpp @@ -47,7 +47,8 @@ nano::store::rocksdb::component::component (nano::logger & logger_a, std::filesy peer_store, confirmation_height_store, final_vote_store, - version_store + version_store, + rep_weight_store }, // clang-format on block_store{ *this }, @@ -60,6 +61,7 @@ nano::store::rocksdb::component::component (nano::logger & logger_a, std::filesy confirmation_height_store{ *this }, final_vote_store{ *this }, version_store{ *this }, + rep_weight_store{ *this }, logger{ logger_a }, constants{ constants }, rocksdb_config{ rocksdb_config_a }, @@ -172,7 +174,8 @@ std::unordered_map nano::store::rocksdb::component:: { "peers", tables::peers }, { "confirmation_height", tables::confirmation_height }, { "pruned", tables::pruned }, - { "final_votes", tables::final_votes } }; + { "final_votes", tables::final_votes }, + { "rep_weights", tables::rep_weights } }; debug_assert (map.size () == all_tables ().size () + 1); return map; @@ -384,6 +387,11 @@ rocksdb::ColumnFamilyOptions nano::store::rocksdb::component::get_cf_options (st std::shared_ptr<::rocksdb::TableFactory> table_factory (::rocksdb::NewBlockBasedTableFactory (get_active_table_options (block_cache_size_bytes * 2))); cf_options = get_active_cf_options (table_factory, memtable_size_bytes); } + else if (cf_name_a == "rep_weights") + { + std::shared_ptr<::rocksdb::TableFactory> table_factory (::rocksdb::NewBlockBasedTableFactory (get_active_table_options (block_cache_size_bytes * 2))); + cf_options = get_active_cf_options (table_factory, memtable_size_bytes); + } else if (cf_name_a == ::rocksdb::kDefaultColumnFamilyName) { // Do nothing. @@ -509,6 +517,8 @@ rocksdb::ColumnFamilyHandle * nano::store::rocksdb::component::table_to_column_f return get_column_family ("confirmation_height"); case tables::final_votes: return get_column_family ("final_votes"); + case tables::rep_weights: + return get_column_family ("rep_weights"); default: release_assert (false); return get_column_family (""); @@ -666,6 +676,14 @@ uint64_t nano::store::rocksdb::component::count (store::transaction const & tran ++sum; } } + // rep_weights should only be used in tests otherwise there can be performance issues. + else if (table_a == tables::rep_weights) + { + for (auto i (rep_weight.begin (transaction_a)), n (rep_weight.end ()); i != n; ++i) + { + ++sum; + } + } else { debug_assert (false); @@ -850,7 +868,7 @@ void nano::store::rocksdb::component::on_flush (::rocksdb::FlushJobInfo const & std::vector nano::store::rocksdb::component::all_tables () const { - return std::vector{ tables::accounts, tables::blocks, tables::confirmation_height, tables::final_votes, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::pruned, tables::vote }; + return std::vector{ tables::accounts, tables::blocks, tables::confirmation_height, tables::final_votes, tables::frontiers, tables::meta, tables::online_weight, tables::peers, tables::pending, tables::pruned, tables::vote, tables::rep_weights }; } bool nano::store::rocksdb::component::copy_db (std::filesystem::path const & destination_path) diff --git a/nano/store/rocksdb/rocksdb.hpp b/nano/store/rocksdb/rocksdb.hpp index 1c51bbbb05..f23145b021 100644 --- a/nano/store/rocksdb/rocksdb.hpp +++ b/nano/store/rocksdb/rocksdb.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -51,6 +52,7 @@ class component : public nano::store::component nano::store::rocksdb::pending pending_store; nano::store::rocksdb::pruned pruned_store; nano::store::rocksdb::version version_store; + nano::store::rocksdb::rep_weight rep_weight_store; public: friend class nano::store::rocksdb::account; @@ -63,6 +65,7 @@ class component : public nano::store::component friend class nano::store::rocksdb::pending; friend class nano::store::rocksdb::pruned; friend class nano::store::rocksdb::version; + friend class nano::store::rocksdb::rep_weight; explicit component (nano::logger &, std::filesystem::path const &, nano::ledger_constants & constants, nano::rocksdb_config const & = nano::rocksdb_config{}, bool open_read_only = false); diff --git a/nano/store/tables.hpp b/nano/store/tables.hpp index e0a03bd020..cf53a0f914 100644 --- a/nano/store/tables.hpp +++ b/nano/store/tables.hpp @@ -18,7 +18,8 @@ enum class tables peers, pending, pruned, - vote + vote, + rep_weights, }; } // namespace nano diff --git a/nano/test_common/CMakeLists.txt b/nano/test_common/CMakeLists.txt index 6271377f45..0f0b9d3c4a 100644 --- a/nano/test_common/CMakeLists.txt +++ b/nano/test_common/CMakeLists.txt @@ -4,6 +4,8 @@ add_library( chains.cpp ledger.hpp ledger.cpp + make_store.hpp + make_store.cpp network.hpp network.cpp rate_observer.cpp diff --git a/nano/test_common/make_store.cpp b/nano/test_common/make_store.cpp new file mode 100644 index 0000000000..9a980d8513 --- /dev/null +++ b/nano/test_common/make_store.cpp @@ -0,0 +1,11 @@ +#include +#include +#include +#include +#include +#include + +std::unique_ptr nano::test::make_store () +{ + return nano::make_store (nano::default_logger (), nano::unique_path (), nano::dev::constants); +} diff --git a/nano/test_common/make_store.hpp b/nano/test_common/make_store.hpp new file mode 100644 index 0000000000..084b66a1fb --- /dev/null +++ b/nano/test_common/make_store.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace nano::store +{ +class component; +} + +namespace nano::test +{ +std::unique_ptr make_store (); +}