Merge branch 'master' into wen_restart_use_appropriate_epoch_stakes

builtins-default-costs/src/lib.rs

@@ -12,6 +12,14 @@ use {
 // Number of compute units for each built-in programs
 lazy_static! {
     /// Number of compute units for each built-in programs
+    ///
+    /// DEVELOPER WARNING: This map CANNOT be modified without causing a
+    /// consensus failure because this map is used to calculate the compute
+    /// limit for transactions that don't specify a compute limit themselves as
+    /// of https://github.com/anza-xyz/agave/issues/2212.  It's also used to
+    /// calculate the cost of a transaction which is used in replay to enforce
+    /// block cost limits as of
+    /// https://github.com/solana-labs/solana/issues/29595.
     pub static ref BUILTIN_INSTRUCTION_COSTS: AHashMap<Pubkey, u64> = [
         (solana_stake_program::id(), solana_stake_program::stake_instruction::DEFAULT_COMPUTE_UNITS),
         (solana_config_program::id(), solana_config_program::config_processor::DEFAULT_COMPUTE_UNITS),
@@ -26,6 +34,7 @@ lazy_static! {
         // Note: These are precompile, run directly in bank during sanitizing;
         (secp256k1_program::id(), 0),
         (ed25519_program::id(), 0),
+        // DO NOT ADD MORE ENTRIES TO THIS MAP
     ]
     .iter()
     .cloned()

cli/src/vote.rs

@@ -31,7 +31,7 @@ use {
     solana_rpc_client_api::config::RpcGetVoteAccountsConfig,
     solana_rpc_client_nonce_utils::blockhash_query::BlockhashQuery,
     solana_sdk::{
-        account::Account, commitment_config::CommitmentConfig, feature, message::Message,
+        account::Account, commitment_config::CommitmentConfig, message::Message,
         native_token::lamports_to_sol, pubkey::Pubkey, system_instruction::SystemError,
         transaction::Transaction,
     },
@@ -819,13 +819,7 @@ pub fn process_create_vote_account(
 
     let fee_payer = config.signers[fee_payer];
     let nonce_authority = config.signers[nonce_authority];
-
-    let is_feature_active = (!sign_only)
-        .then(solana_sdk::feature_set::vote_state_add_vote_latency::id)
-        .and_then(|feature_address| rpc_client.get_account(&feature_address).ok())
-        .and_then(|account| feature::from_account(&account))
-        .map_or(false, |feature| feature.activated_at.is_some());
-    let space = VoteStateVersions::vote_state_size_of(is_feature_active) as u64;
+    let space = VoteStateVersions::vote_state_size_of(true) as u64;
 
     let build_message = |lamports| {
         let vote_init = VoteInit {

core/src/banking_stage/latest_unprocessed_votes.rs

@@ -8,9 +8,13 @@ use {
     solana_perf::packet::Packet,
     solana_runtime::bank::Bank,
     solana_sdk::{
+        account::from_account,
         clock::{Slot, UnixTimestamp},
+        hash::Hash,
         program_utils::limited_deserialize,
         pubkey::Pubkey,
+        slot_hashes::SlotHashes,
+        sysvar,
     },
     solana_vote_program::vote_instruction::VoteInstruction,
     std::{
@@ -36,6 +40,7 @@ pub struct LatestValidatorVotePacket {
     pubkey: Pubkey,
     vote: Option<Arc<ImmutableDeserializedPacket>>,
     slot: Slot,
+    hash: Hash,
     forwarded: bool,
     timestamp: Option<UnixTimestamp>,
 }
@@ -70,11 +75,13 @@ impl LatestValidatorVotePacket {
                     .first()
                     .ok_or(DeserializedPacketError::VoteTransactionError)?;
                 let slot = vote_state_update_instruction.last_voted_slot().unwrap_or(0);
+                let hash = vote_state_update_instruction.hash();
                 let timestamp = vote_state_update_instruction.timestamp();
 
                 Ok(Self {
                     vote: Some(vote),
                     slot,
+                    hash,
                     pubkey,
                     vote_source,
                     forwarded: false,
@@ -97,6 +104,10 @@ impl LatestValidatorVotePacket {
         self.slot
     }
 
+    pub(crate) fn hash(&self) -> Hash {
+        self.hash
+    }
+
     pub fn timestamp(&self) -> Option<UnixTimestamp> {
         self.timestamp
     }
@@ -115,9 +126,6 @@ impl LatestValidatorVotePacket {
     }
 }
 
-// TODO: replace this with rand::seq::index::sample_weighted once we can update rand to 0.8+
-// This requires updating dependencies of ed25519-dalek as rand_core is not compatible cross
-// version https://github.com/dalek-cryptography/ed25519-dalek/pull/214
 pub(crate) fn weighted_random_order_by_stake<'a>(
     bank: &Bank,
     pubkeys: impl Iterator<Item = &'a Pubkey>,
@@ -322,17 +330,30 @@ impl LatestUnprocessedVotes {
     }
 
     /// Drains all votes yet to be processed sorted by a weighted random ordering by stake
+    /// Do not touch votes that are for a different fork from `bank` as we know they will fail,
+    /// however the next bank could be built on a different fork and consume these votes.
     pub fn drain_unprocessed(&self, bank: Arc<Bank>) -> Vec<Arc<ImmutableDeserializedPacket>> {
-        let pubkeys_by_stake = weighted_random_order_by_stake(
-            &bank,
-            self.latest_votes_per_pubkey.read().unwrap().keys(),
-        )
-        .collect_vec();
+        let slot_hashes = bank
+            .get_account(&sysvar::slot_hashes::id())
+            .and_then(|account| from_account::<SlotHashes, _>(&account));
+        if slot_hashes.is_none() {
+            error!(
+                "Slot hashes sysvar doesn't exist on bank {}. Including all votes without filtering",
+                bank.slot()
+            );
+        }
+
+        let pubkeys_by_stake = {
+            let binding = self.latest_votes_per_pubkey.read().unwrap();
+            weighted_random_order_by_stake(&bank, binding.keys())
+        };
         pubkeys_by_stake
-            .into_iter()
             .filter_map(|pubkey| {
                 self.get_entry(pubkey).and_then(|lock| {
                     let mut latest_vote = lock.write().unwrap();
+                    if !Self::is_valid_for_our_fork(&latest_vote, &slot_hashes) {
+                        return None;
+                    }
                     latest_vote.take_vote().map(|vote| {
                         self.num_unprocessed_votes.fetch_sub(1, Ordering::Relaxed);
                         vote
@@ -342,6 +363,21 @@ impl LatestUnprocessedVotes {
             .collect_vec()
     }
 
+    /// Check if `vote` can land in our fork based on `slot_hashes`
+    fn is_valid_for_our_fork(
+        vote: &LatestValidatorVotePacket,
+        slot_hashes: &Option<SlotHashes>,
+    ) -> bool {
+        let Some(slot_hashes) = slot_hashes else {
+            // When slot hashes is not present we do not filter
+            return true;
+        };
+        slot_hashes
+            .get(&vote.slot())
+            .map(|found_hash| *found_hash == vote.hash())
+            .unwrap_or(false)
+    }
+
     /// Sometimes we forward and hold the packets, sometimes we forward and clear.
     /// This also clears all gossip votes since by definition they have been forwarded
     pub fn clear_forwarded_packets(&self) {

ledger/src/blockstore.rs

@@ -34,9 +34,7 @@ use {
     rand::Rng,
     rayon::iter::{IntoParallelIterator, ParallelIterator},
     rocksdb::{DBRawIterator, LiveFile},
-    solana_accounts_db::hardened_unpack::{
-        unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
-    },
+    solana_accounts_db::hardened_unpack::unpack_genesis_archive,
     solana_entry::entry::{create_ticks, Entry},
     solana_measure::measure::Measure,
     solana_metrics::{
@@ -4963,6 +4961,22 @@ macro_rules! create_new_tmp_ledger {
         $crate::blockstore::create_new_ledger_from_name(
             $crate::tmp_ledger_name!(),
             $genesis_config,
+            $crate::macro_reexports::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
+            $crate::blockstore_options::LedgerColumnOptions::default(),
+        )
+    };
+}
+
+#[macro_export]
+macro_rules! create_new_tmp_ledger_with_size {
+    (
+        $genesis_config:expr,
+        $max_genesis_archive_unpacked_size:expr $(,)?
+    ) => {
+        $crate::blockstore::create_new_ledger_from_name(
+            $crate::tmp_ledger_name!(),
+            $genesis_config,
+            $max_genesis_archive_unpacked_size,
             $crate::blockstore_options::LedgerColumnOptions::default(),
         )
     };
@@ -4974,6 +4988,7 @@ macro_rules! create_new_tmp_ledger_fifo {
         $crate::blockstore::create_new_ledger_from_name(
             $crate::tmp_ledger_name!(),
             $genesis_config,
+            $crate::macro_reexports::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
             $crate::blockstore_options::LedgerColumnOptions {
                 shred_storage_type: $crate::blockstore_options::ShredStorageType::RocksFifo(
                     $crate::blockstore_options::BlockstoreRocksFifoOptions::new_for_tests(),
@@ -4990,6 +5005,7 @@ macro_rules! create_new_tmp_ledger_auto_delete {
         $crate::blockstore::create_new_ledger_from_name_auto_delete(
             $crate::tmp_ledger_name!(),
             $genesis_config,
+            $crate::macro_reexports::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
             $crate::blockstore_options::LedgerColumnOptions::default(),
         )
     };
@@ -5001,6 +5017,7 @@ macro_rules! create_new_tmp_ledger_fifo_auto_delete {
         $crate::blockstore::create_new_ledger_from_name_auto_delete(
             $crate::tmp_ledger_name!(),
             $genesis_config,
+            $crate::macro_reexports::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
             $crate::blockstore_options::LedgerColumnOptions {
                 shred_storage_type: $crate::blockstore_options::ShredStorageType::RocksFifo(
                     $crate::blockstore_options::BlockstoreRocksFifoOptions::new_for_tests(),
@@ -5027,10 +5044,15 @@ pub(crate) fn verify_shred_slots(slot: Slot, parent: Slot, root: Slot) -> bool {
 pub fn create_new_ledger_from_name(
     name: &str,
     genesis_config: &GenesisConfig,
+    max_genesis_archive_unpacked_size: u64,
     column_options: LedgerColumnOptions,
 ) -> (PathBuf, Hash) {
-    let (ledger_path, blockhash) =
-        create_new_ledger_from_name_auto_delete(name, genesis_config, column_options);
+    let (ledger_path, blockhash) = create_new_ledger_from_name_auto_delete(
+        name,
+        genesis_config,
+        max_genesis_archive_unpacked_size,
+        column_options,
+    );
     (ledger_path.into_path(), blockhash)
 }
 
@@ -5041,13 +5063,14 @@ pub fn create_new_ledger_from_name(
 pub fn create_new_ledger_from_name_auto_delete(
     name: &str,
     genesis_config: &GenesisConfig,
+    max_genesis_archive_unpacked_size: u64,
     column_options: LedgerColumnOptions,
 ) -> (TempDir, Hash) {
     let ledger_path = get_ledger_path_from_name_auto_delete(name);
     let blockhash = create_new_ledger(
         ledger_path.path(),
         genesis_config,
-        MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
+        max_genesis_archive_unpacked_size,
         column_options,
     )
     .unwrap();

ledger/src/lib.rs

@@ -46,3 +46,8 @@ extern crate log;
 #[cfg_attr(feature = "frozen-abi", macro_use)]
 #[cfg(feature = "frozen-abi")]
 extern crate solana_frozen_abi_macro;
+
+#[doc(hidden)]
+pub mod macro_reexports {
+    pub use solana_accounts_db::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE;
+}

local-cluster/src/local_cluster.rs

@@ -18,7 +18,7 @@ use {
         contact_info::{ContactInfo, Protocol},
         gossip_service::discover_cluster,
     },
-    solana_ledger::{create_new_tmp_ledger, shred::Shred},
+    solana_ledger::{create_new_tmp_ledger_with_size, shred::Shred},
     solana_rpc_client::rpc_client::RpcClient,
     solana_runtime::{
         genesis_utils::{
@@ -32,7 +32,6 @@ use {
         clock::{Slot, DEFAULT_DEV_SLOTS_PER_EPOCH, DEFAULT_TICKS_PER_SLOT, MAX_PROCESSING_AGE},
         commitment_config::CommitmentConfig,
         epoch_schedule::EpochSchedule,
-        feature_set,
         genesis_config::{ClusterType, GenesisConfig},
         message::Message,
         poh_config::PohConfig,
@@ -312,9 +311,13 @@ impl LocalCluster {
             .native_instruction_processors
             .extend_from_slice(&config.native_instruction_processors);
 
-        let (leader_ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_config);
-        let leader_contact_info = leader_node.info.clone();
         let mut leader_config = safe_clone_config(&config.validator_configs[0]);
+        let (leader_ledger_path, _blockhash) = create_new_tmp_ledger_with_size!(
+            &genesis_config,
+            leader_config.max_genesis_archive_unpacked_size,
+        );
+
+        let leader_contact_info = leader_node.info.clone();
         leader_config.rpc_addrs = Some((
             leader_node.info.rpc().unwrap(),
             leader_node.info.rpc_pubsub().unwrap(),
@@ -494,7 +497,10 @@ impl LocalCluster {
         let validator_pubkey = validator_keypair.pubkey();
         let validator_node = Node::new_localhost_with_pubkey(&validator_keypair.pubkey());
         let contact_info = validator_node.info.clone();
-        let (ledger_path, _blockhash) = create_new_tmp_ledger!(&self.genesis_config);
+        let (ledger_path, _blockhash) = create_new_tmp_ledger_with_size!(
+            &self.genesis_config,
+            validator_config.max_genesis_archive_unpacked_size,
+        );
 
         // Give the validator some lamports to setup vote accounts
         if is_listener {
@@ -771,18 +777,6 @@ impl LocalCluster {
             == 0
         {
             // 1) Create vote account
-            // Unlike the bootstrap validator we have to check if the new vote state is being used
-            // as the cluster is already running, and using the wrong account size will cause the
-            // InitializeAccount tx to fail
-            let use_current_vote_state = client
-                .rpc_client()
-                .poll_get_balance_with_commitment(
-                    &feature_set::vote_state_add_vote_latency::id(),
-                    CommitmentConfig::processed(),
-                )
-                .unwrap_or(0)
-                > 0;
-
             let instructions = vote_instruction::create_account_with_config(
                 &from_account.pubkey(),
                 &vote_account_pubkey,
@@ -794,8 +788,7 @@ impl LocalCluster {
                 },
                 amount,
                 vote_instruction::CreateVoteAccountConfig {
-                    space: vote_state::VoteStateVersions::vote_state_size_of(use_current_vote_state)
-                        as u64,
+                    space: vote_state::VoteStateVersions::vote_state_size_of(true) as u64,
                     ..vote_instruction::CreateVoteAccountConfig::default()
                 },
             );