Make reth data pipeline privacy aware (#1)

* Added leaf node privacy flag. Default to false * default * add default keyword * working branch node with privacy flag * made changes for storageSlot; TODO: need to change EVM * changed how hasing storage is done * baseline point for working reth. need to work on channeling changes to include is_private flag * passed test * latest * latest * working revm-inspctors * debugging * debugging * debugging * debugging * build completed * test compiled * removing comment * rm comment * Use seismic's versions of alloy & revm dependencies (#2) * patch correct versions of seismic-reth * patch correct versions, builds * comment out alloy-core * test compiled * addressed comments * get rid of warnings * fix warnings * make test succeed * correct cargo.toml * restores cargo.toml * restores cargo.toml * added tests * fix fomrat --------- Co-authored-by: SFYLL <sd@seismic.systems> Co-authored-by: Christian Drappi <c@seismic.systems>
SeismicSystems · Sep 24, 2024 · 8799068 · 8799068
1 parent 63c126f
commit 8799068
Show file tree

Hide file tree

Showing 57 changed files with 474 additions and 340 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -146,6 +146,7 @@ members = [
 ]
 default-members = ["bin/reth"]
 
+
 # Explicitly set the resolver to version 2, which is the default for packages with edition >= 2021
 # https://doc.rust-lang.org/edition-guide/rust-2021/default-cargo-resolver.html
 resolver = "2"
@@ -398,7 +399,6 @@ reth-trie-parallel = { path = "crates/trie/parallel" }
 revm = { version = "14.0.0", features = [
     "std",
     "secp256k1",
-    "blst",
 ], default-features = false }
 revm-inspectors = "0.6"
 revm-primitives = { version = "9.0.0", features = [
@@ -572,6 +572,9 @@ test-fuzz = "5"
 revm = { path = "../seismic-revm/crates/revm" }
 revm-primitives = { path = "../seismic-revm/crates/primitives" }
 
+# seismic-revm-inspectors
+revm-inspectors = { path = "../seismic-revm-inspectors" }
+
 # seismic-alloy-core
 # alloy-core = { path = "../seismic-alloy-core/crates/core" }
 alloy-dyn-abi = { path = "../seismic-alloy-core/crates/dyn-abi" }

diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs
@@ -810,13 +810,14 @@ mod tests {
     use rand::Rng;
     use reth_errors::ProviderResult;
     use reth_primitives::{
-        Account, BlockNumber, Bytecode, Bytes, Receipt, Requests, StorageKey, StorageValue,
+        Account, BlockNumber, Bytecode, Bytes, Receipt, Requests, StorageKey,
     };
     use reth_storage_api::{
         AccountReader, BlockHashReader, StateProofProvider, StateProvider, StateRootProvider,
         StorageRootProvider,
     };
     use reth_trie::{prefix_set::TriePrefixSetsMut, AccountProof, HashedStorage};
+    use revm::primitives::FlaggedStorage;
 
     fn create_mock_state(
         test_block_builder: &mut TestBlockBuilder,
@@ -856,7 +857,7 @@ mod tests {
             &self,
             _address: Address,
             _storage_key: StorageKey,
-        ) -> ProviderResult<Option<StorageValue>> {
+        ) -> ProviderResult<Option<FlaggedStorage>> {
             Ok(None)
         }
 

diff --git a/crates/chain-state/src/memory_overlay.rs b/crates/chain-state/src/memory_overlay.rs
@@ -1,7 +1,8 @@
 use super::ExecutedBlock;
 use reth_errors::ProviderResult;
 use reth_primitives::{
-    keccak256, Account, Address, BlockNumber, Bytecode, Bytes, StorageKey, StorageValue, B256,
+    keccak256, Account, Address, BlockNumber, Bytecode, Bytes, StorageKey, B256,
+    revm_primitives::FlaggedStorage,
 };
 use reth_storage_api::{
     AccountReader, BlockHashReader, StateProofProvider, StateProvider, StateProviderBox,
@@ -185,7 +186,7 @@ impl StateProvider for MemoryOverlayStateProvider {
         &self,
         address: Address,
         storage_key: StorageKey,
-    ) -> ProviderResult<Option<StorageValue>> {
+    ) -> ProviderResult<Option<FlaggedStorage>> {
         for block in &self.in_memory {
             if let Some(value) = block.execution_output.storage(&address, storage_key.into()) {
                 return Ok(Some(value))

diff --git a/crates/ethereum/evm/src/execute.rs b/crates/ethereum/evm/src/execute.rs
@@ -29,7 +29,7 @@ use reth_revm::{
 };
 use revm_primitives::{
     db::{Database, DatabaseCommit},
-    BlockEnv, CfgEnvWithHandlerCfg, EVMError, EnvWithHandlerCfg, ResultAndState,
+    BlockEnv, CfgEnvWithHandlerCfg, EVMError, EnvWithHandlerCfg, ResultAndState, FlaggedStorage
 };
 
 #[cfg(not(feature = "std"))]
@@ -604,14 +604,14 @@ mod tests {
         // get timestamp storage and compare
         let timestamp_storage =
             executor.state.storage(BEACON_ROOTS_ADDRESS, U256::from(timestamp_index)).unwrap();
-        assert_eq!(timestamp_storage, U256::from(header.timestamp));
+        assert_eq!(timestamp_storage, FlaggedStorage::new_from_value(header.timestamp));
 
         // get parent beacon block root storage and compare
         let parent_beacon_block_root_storage = executor
             .state
             .storage(BEACON_ROOTS_ADDRESS, U256::from(parent_beacon_block_root_index))
             .expect("storage value should exist");
-        assert_eq!(parent_beacon_block_root_storage, U256::from(0x69));
+        assert_eq!(parent_beacon_block_root_storage, FlaggedStorage::new_from_value(0x69));
     }
 
     #[test]
@@ -857,14 +857,14 @@ mod tests {
             .state_mut()
             .storage(BEACON_ROOTS_ADDRESS, U256::from(timestamp_index))
             .unwrap();
-        assert_eq!(timestamp_storage, U256::from(header.timestamp));
+        assert_eq!(timestamp_storage, FlaggedStorage::new_from_value(header.timestamp));
 
         // get parent beacon block root storage and compare
         let parent_beacon_block_root_storage = executor
             .state_mut()
             .storage(BEACON_ROOTS_ADDRESS, U256::from(parent_beacon_block_root_index))
             .unwrap();
-        assert_eq!(parent_beacon_block_root_storage, U256::from(0x69));
+        assert_eq!(parent_beacon_block_root_storage, FlaggedStorage::new_from_value(0x69));
     }
 
     /// Create a state provider with blockhashes and the EIP-2935 system contract.
@@ -1043,7 +1043,7 @@ mod tests {
                 .state_mut()
                 .storage(HISTORY_STORAGE_ADDRESS, U256::from(fork_activation_block - 1))
                 .unwrap(),
-            U256::ZERO
+            FlaggedStorage::ZERO
         );
 
         // the hash of the block itself should not be in storage
@@ -1109,7 +1109,7 @@ mod tests {
                     U256::from(fork_activation_block % BLOCKHASH_SERVE_WINDOW as u64 - 1)
                 )
                 .unwrap(),
-            U256::ZERO
+            FlaggedStorage::ZERO
         );
     }
 
@@ -1199,7 +1199,7 @@ mod tests {
         assert!(executor.state_mut().basic(HISTORY_STORAGE_ADDRESS).unwrap().is_some());
         assert_ne!(
             executor.state_mut().storage(HISTORY_STORAGE_ADDRESS, U256::ZERO).unwrap(),
-            U256::ZERO
+            FlaggedStorage::ZERO
         );
         assert!(executor
             .state_mut()
@@ -1241,11 +1241,11 @@ mod tests {
         assert!(executor.state_mut().basic(HISTORY_STORAGE_ADDRESS).unwrap().is_some());
         assert_ne!(
             executor.state_mut().storage(HISTORY_STORAGE_ADDRESS, U256::ZERO).unwrap(),
-            U256::ZERO
+            FlaggedStorage::ZERO
         );
         assert_ne!(
             executor.state_mut().storage(HISTORY_STORAGE_ADDRESS, U256::from(1)).unwrap(),
-            U256::ZERO
+            FlaggedStorage::ZERO
         );
         assert!(executor
             .state_mut()

diff --git a/crates/evm/execution-types/src/execution_outcome.rs b/crates/evm/execution-types/src/execution_outcome.rs
@@ -6,7 +6,7 @@ use reth_primitives::{
 use reth_trie::HashedPostState;
 use revm::{
     db::{states::BundleState, BundleAccount},
-    primitives::AccountInfo,
+    primitives::{AccountInfo, FlaggedStorage},
 };
 use std::collections::HashMap;
 
@@ -56,7 +56,7 @@ pub struct ExecutionOutcome {
 
 /// Type used to initialize revms bundle state.
 pub type BundleStateInit =
-    HashMap<Address, (Option<Account>, Option<Account>, HashMap<B256, (U256, U256)>)>;
+    HashMap<Address, (Option<Account>, Option<Account>, HashMap<B256, ((U256, bool), (U256, bool))>)>;
 
 /// Types used inside `RevertsInit` to initialize revms reverts.
 pub type AccountRevertInit = (Option<Option<Account>>, Vec<StorageEntry>);
@@ -94,14 +94,14 @@ impl ExecutionOutcome {
         let mut reverts = revert_init.into_iter().collect::<Vec<_>>();
         reverts.sort_unstable_by_key(|a| a.0);
 
-        // initialize revm bundle
+        // initialize revm bundle 
         let bundle = BundleState::new(
             state_init.into_iter().map(|(address, (original, present, storage))| {
                 (
                     address,
                     original.map(Into::into),
                     present.map(Into::into),
-                    storage.into_iter().map(|(k, s)| (k.into(), s)).collect(),
+                    storage.into_iter().map(|(k, (orig_value, new_value))| (k.into(), (FlaggedStorage::new_from_tuple(orig_value), FlaggedStorage::new_from_tuple(new_value)))).collect(),
                 )
             }),
             reverts.into_iter().map(|(_, reverts)| {
@@ -110,7 +110,7 @@ impl ExecutionOutcome {
                     (
                         address,
                         original.map(|i| i.map(Into::into)),
-                        storage.into_iter().map(|entry| (entry.key.into(), entry.value)),
+                        storage.into_iter().map(|entry| (entry.key.into(), FlaggedStorage {value: entry.value, is_private: entry.is_private})),
                     )
                 })
             }),
@@ -153,7 +153,7 @@ impl ExecutionOutcome {
     /// Get storage if value is known.
     ///
     /// This means that depending on status we can potentially return `U256::ZERO`.
-    pub fn storage(&self, address: &Address, storage_key: U256) -> Option<U256> {
+    pub fn storage(&self, address: &Address, storage_key: U256) -> Option<FlaggedStorage> {
         self.bundle.account(address).and_then(|a| a.storage_slot(storage_key))
     }
 

diff --git a/crates/payload/builder/src/database.rs b/crates/payload/builder/src/database.rs
@@ -3,7 +3,7 @@
 use reth_primitives::{
     revm_primitives::{
         db::{Database, DatabaseRef},
-        AccountInfo, Address, Bytecode, B256,
+        AccountInfo, Address, Bytecode, B256, FlaggedStorage
     },
     U256,
 };
@@ -55,7 +55,7 @@ impl CachedReads {
         &mut self,
         address: Address,
         info: AccountInfo,
-        storage: HashMap<U256, U256>,
+        storage: HashMap<U256, FlaggedStorage>,
     ) {
         self.accounts.insert(address, CachedAccount { info: Some(info), storage });
     }
@@ -91,7 +91,7 @@ impl<'a, DB: DatabaseRef> Database for CachedReadsDbMut<'a, DB> {
         Ok(code)
     }
 
-    fn storage(&mut self, address: Address, index: U256) -> Result<U256, Self::Error> {
+    fn storage(&mut self, address: Address, index: U256) -> Result<FlaggedStorage, Self::Error> {
         match self.cached.accounts.entry(address) {
             Entry::Occupied(mut acc_entry) => match acc_entry.get_mut().storage.entry(index) {
                 Entry::Occupied(entry) => Ok(*entry.get()),
@@ -106,7 +106,7 @@ impl<'a, DB: DatabaseRef> Database for CachedReadsDbMut<'a, DB> {
                     account.storage.insert(index, value);
                     (account, value)
                 } else {
-                    (CachedAccount::new(info), U256::ZERO)
+                    (CachedAccount::new(info), FlaggedStorage::ZERO)
                 };
                 acc_entry.insert(account);
                 Ok(value)
@@ -144,7 +144,7 @@ impl<'a, DB: DatabaseRef> DatabaseRef for CachedReadsDBRef<'a, DB> {
         self.inner.borrow_mut().code_by_hash(code_hash)
     }
 
-    fn storage_ref(&self, address: Address, index: U256) -> Result<U256, Self::Error> {
+    fn storage_ref(&self, address: Address, index: U256) -> Result<FlaggedStorage, Self::Error> {
         self.inner.borrow_mut().storage(address, index)
     }
 
@@ -156,7 +156,7 @@ impl<'a, DB: DatabaseRef> DatabaseRef for CachedReadsDBRef<'a, DB> {
 #[derive(Debug, Clone)]
 struct CachedAccount {
     info: Option<AccountInfo>,
-    storage: HashMap<U256, U256>,
+    storage: HashMap<U256, FlaggedStorage>,
 }
 
 impl CachedAccount {

diff --git a/crates/primitives-traits/src/storage.rs b/crates/primitives-traits/src/storage.rs
@@ -1,5 +1,6 @@
 use alloy_primitives::{B256, U256};
 use reth_codecs::{add_arbitrary_tests, Compact};
+use revm_primitives::FlaggedStorage;
 use serde::{Deserialize, Serialize};
 
 /// Account storage entry.
@@ -13,18 +14,40 @@ pub struct StorageEntry {
     pub key: B256,
     /// Value on storage key.
     pub value: U256,
+    /// Indicates whether the value is private
+    pub is_private: bool,
 }
 
 impl StorageEntry {
     /// Create a new `StorageEntry` with given key and value.
-    pub const fn new(key: B256, value: U256) -> Self {
-        Self { key, value }
+    pub const fn new(key: B256, value: U256, is_private: bool) -> Self {
+        Self { key, value, is_private }
+    }
+
+    /// Convert the storage entry to a flagged storage entry. 
+    pub const fn to_flagged_storage(self) -> FlaggedStorage {
+        FlaggedStorage { value: self.value, is_private: self.is_private }
+    }
+}
+
+impl From<(B256, U256, bool)> for StorageEntry {
+    fn from((key, value, is_private): (B256, U256, bool)) -> Self {
+        Self { key, value, is_private }
+    }
+}
+
+impl From<(B256, (U256, bool))> for StorageEntry {
+    fn from((key, (value, is_private)): (B256, (U256, bool))) -> Self {
+        Self { key, value, is_private }
     }
 }
 
-impl From<(B256, U256)> for StorageEntry {
-    fn from((key, value): (B256, U256)) -> Self {
-        Self { key, value }
+impl From<StorageEntry> for FlaggedStorage {
+    fn from(entry: StorageEntry) -> Self {
+        Self {
+            value: entry.value,
+            is_private: entry.is_private,
+        }
     }
 }
 
@@ -38,12 +61,14 @@ impl Compact for StorageEntry {
     {
         // for now put full bytes and later compress it.
         buf.put_slice(&self.key[..]);
-        self.value.to_compact(buf) + 32
+        buf.put_u8(self.is_private as u8);
+        self.value.to_compact(buf) + 32 + 1
     }
 
     fn from_compact(buf: &[u8], len: usize) -> (Self, &[u8]) {
         let key = B256::from_slice(&buf[..32]);
-        let (value, out) = U256::from_compact(&buf[32..], len - 32);
-        (Self { key, value }, out)
+        let is_private = buf[32] != 0;
+        let (value, out) = U256::from_compact(&buf[33..], len - 33);
+        (Self { key, value, is_private }, out)
     }
 }