replace account index bulk insert with iterator (backport #18198)

runtime/src/accounts_db.rs

@@ -1396,7 +1396,7 @@ impl Default for AccountsDb {
 }
 
 type GenerateIndexAccountsMap<'a> =
-    HashMap<Pubkey, (StoredMetaWriteVersion, AppendVecId, StoredAccountMeta<'a>)>;
+    HashMap<&'a Pubkey, (StoredMetaWriteVersion, AppendVecId, StoredAccountMeta<'a>)>;
 
 impl AccountsDb {
     pub fn new(paths: Vec<PathBuf>, cluster_type: &ClusterType) -> Self {
@@ -5854,7 +5854,7 @@ impl AccountsDb {
             let accounts = storage.all_accounts();
             accounts.into_iter().for_each(|stored_account| {
                 let this_version = stored_account.meta.write_version;
-                match accounts_map.entry(stored_account.meta.pubkey) {
+                match accounts_map.entry(&stored_account.meta.pubkey) {
                     std::collections::hash_map::Entry::Vacant(entry) => {
                         entry.insert((this_version, storage.append_vec_id(), stored_account));
                     }
@@ -5881,24 +5881,24 @@ impl AccountsDb {
             return 0;
         }
 
+        let len = accounts_map.len();
         let items = accounts_map
             .iter()
             .map(|(pubkey, (_, store_id, stored_account))| {
                 (
-                    pubkey,
+                    *pubkey,
                     AccountInfo {
                         store_id: *store_id,
                         offset: stored_account.offset,
                         stored_size: stored_account.stored_size,
                         lamports: stored_account.account_meta.lamports,
                     },
                 )
-            })
-            .collect::<Vec<_>>();
+            });
 
         let (dirty_pubkeys, insert_us) = self
             .accounts_index
-            .insert_new_if_missing_into_primary_index(*slot, items);
+            .insert_new_if_missing_into_primary_index(*slot, len, items);
 
         // dirty_pubkeys will contain a pubkey if an item has multiple rooted entries for
         // a given pubkey. If there is just a single item, there is no cleaning to

runtime/src/accounts_index.rs

@@ -1355,15 +1355,14 @@ impl<T: 'static + Clone + IsCached + ZeroLamport> AccountsIndex<T> {
     // But, does NOT update secondary index
     // This is designed to be called at startup time.
     #[allow(clippy::needless_collect)]
-    pub(crate) fn insert_new_if_missing_into_primary_index(
-        &self,
+    pub(crate) fn insert_new_if_missing_into_primary_index<'a>(
+        &'a self,
         slot: Slot,
-        items: Vec<(&Pubkey, T)>,
+        item_len: usize,
+        items: impl Iterator<Item = (&'a Pubkey, T)>,
     ) -> (Vec<Pubkey>, u64) {
         // returns (duplicate pubkey mask, insertion time us)
-        let item_len = items.len();
         let potentially_new_items = items
-            .into_iter()
             .map(|(pubkey, account_info)| {
                 // this value is equivalent to what update() below would have created if we inserted a new item
                 (
@@ -2523,7 +2522,7 @@ pub mod tests {
         let index = AccountsIndex::<bool>::default();
         let account_info = true;
         let items = vec![(pubkey, account_info)];
-        index.insert_new_if_missing_into_primary_index(slot, items);
+        index.insert_new_if_missing_into_primary_index(slot, items.len(), items.into_iter());
 
         let mut ancestors = Ancestors::default();
         assert!(index.get(pubkey, Some(&ancestors), None).is_none());
@@ -2542,7 +2541,7 @@ pub mod tests {
         let index = AccountsIndex::<AccountInfoTest>::default();
         let account_info: AccountInfoTest = 0 as AccountInfoTest;
         let items = vec![(pubkey, account_info)];
-        index.insert_new_if_missing_into_primary_index(slot, items);
+        index.insert_new_if_missing_into_primary_index(slot, items.len(), items.into_iter());
 
         let mut ancestors = Ancestors::default();
         assert!(index.get(pubkey, Some(&ancestors), None).is_none());
@@ -2593,10 +2592,8 @@ pub mod tests {
         let index = AccountsIndex::<bool>::default();
         let account_infos = [true, false];
 
-        index.insert_new_if_missing_into_primary_index(
-            slot0,
-            vec![(&key0, account_infos[0]), (&key1, account_infos[1])],
-        );
+        let items = vec![(&key0, account_infos[0]), (&key1, account_infos[1])];
+        index.insert_new_if_missing_into_primary_index(slot0, items.len(), items.into_iter());
 
         for (i, key) in [key0, key1].iter().enumerate() {
             let entry = index.get_account_read_entry(key).unwrap();
@@ -2631,10 +2628,8 @@ pub mod tests {
                 &mut gc,
             );
         } else {
-            index.insert_new_if_missing_into_primary_index(
-                slot0,
-                vec![(&key, account_infos[0].clone())],
-            );
+            let items = vec![(&key, account_infos[0].clone())];
+            index.insert_new_if_missing_into_primary_index(slot0, items.len(), items.into_iter());
         }
         assert!(gc.is_empty());
 
@@ -2667,10 +2662,8 @@ pub mod tests {
                 &mut gc,
             );
         } else {
-            index.insert_new_if_missing_into_primary_index(
-                slot1,
-                vec![(&key, account_infos[1].clone())],
-            );
+            let items = vec![(&key, account_infos[1].clone())];
+            index.insert_new_if_missing_into_primary_index(slot1, items.len(), items.into_iter());
         }
         assert!(gc.is_empty());