storage: integration of jmt@0.4.0 & reverse index for jmt key hashes

app/Cargo.toml

@@ -24,8 +24,8 @@ penumbra-tct = { path = "../tct" }
 penumbra-proof-params = { path = "../proof-params" }
 
 # Penumbra dependencies
-decaf377 = { version = "0.4" }
-jmt = "0.3"
+decaf377 = "0.4"
+jmt = "0.5"
 tokio = { version = "1.21.1", features = ["full", "tracing"] }
 async-trait = "0.1.52"
 tonic = "0.8.1"
@@ -53,11 +53,12 @@ prost = "0.11"
 rand_chacha = "0.3"
 parking_lot = "0.12"
 
-tendermint = "0.29.0"
-tendermint-proto = "0.29.0"
-tendermint-light-client-verifier = "0.29.0"
-ibc-types = { git = "https://github.com/penumbra-zone/ibc-types", branch = "0.1.x",  version = "0.1.1" }
-ibc-proto = "0.26"
+tendermint = "0.31.0"
+tendermint-proto = "0.31.0"
+tendermint-light-client-verifier = "0.31.0"
+ibc-types = { git = "https://github.com/penumbra-zone/ibc-types", branch = "0.2.x" }
+ibc-proto = { git = "https://github.com/penumbra-zone/ibc-proto-rs", branch = "penumbra" }
+
 
 [dev-dependencies]
 ed25519-consensus = "2"

app/src/dex/component.rs

@@ -11,7 +11,7 @@ use penumbra_crypto::{
 };
 use penumbra_proto::{StateReadProto, StateWriteProto};
 use penumbra_storage::{StateRead, StateWrite};
-use tendermint::abci;
+use tendermint::v0_34::abci;
 use tracing::instrument;
 
 use super::{

chain/Cargo.toml

@@ -14,11 +14,11 @@ penumbra-tct = { path = "../tct" }
 
 # Penumbra dependencies
 decaf377 = "0.4"
+jmt = "0.5"
 
-tendermint = "0.29.0"
-ibc-types = { git = "https://github.com/penumbra-zone/ibc-types", branch = "0.1.x",  version = "0.1.1" }
-
-ics23 = "0.9.0"
+tendermint = "0.31.1"
+ibc-types = { git = "https://github.com/penumbra-zone/ibc-types", branch = "0.2.x" }
+ics23 = "0.10.0"
 
 # Crates.io deps
 ark-ff = { version = "0.4", default_features = false }

chain/src/component/app_hash.rs

@@ -1,3 +1,4 @@
+use anyhow::anyhow;
 use async_trait::async_trait;
 use ibc_types::core::ics23_commitment::merkle::MerkleProof;
 use ibc_types::core::ics23_commitment::{commitment::CommitmentPrefix, specs::ProofSpecs};
@@ -61,6 +62,7 @@ fn apphash_spec() -> ics23::ProofSpec {
         }),
         min_depth: 0,
         max_depth: 1,
+        prehash_key_before_comparison: true,
     }
 }
 
@@ -89,11 +91,14 @@ impl AppHashRead for Snapshot {
         key: Vec<u8>,
     ) -> anyhow::Result<(Vec<u8>, MerkleProof)> {
         let (value, jmt_proof) = self.get_with_proof(key.clone()).await?;
+
+        // TODO(erwan): will immediately follow-up this pr with one that changes
+        // signature to Option<Vec<u8>> here as well, and a `get_without_proof`.
+        // For now, we conserve the semantics of error-ing out on a missing key.
+        let value = value.ok_or_else(|| anyhow!("key not found"))?;
+
         let jmt_root = self.root_hash().await?;
 
-        let jmt_commitment_proof = ics23::CommitmentProof {
-            proof: Some(ics23::commitment_proof::Proof::Exist(jmt_proof)),
-        };
         let root_proof = ics23::CommitmentProof {
             proof: Some(ics23::commitment_proof::Proof::Exist(
                 ics23::ExistenceProof {
@@ -108,7 +113,7 @@ impl AppHashRead for Snapshot {
         Ok((
             value,
             MerkleProof {
-                proofs: vec![jmt_commitment_proof, root_proof],
+                proofs: vec![jmt_proof, root_proof],
             },
         ))
     }

crates/compact-block/Cargo.toml

@@ -33,7 +33,7 @@ serde = { version = "1", features = ["derive"] }
 tracing = "0.1"
 anyhow = "1"
 async-trait = "0.1.52"
-tendermint = "0.29.0"
+tendermint = "0.31.1"
 blake2b_simd = "0.5"
 bytes = "1"
 rand_core = { version = "0.6.3", features = ["getrandom"] }

crates/component/Cargo.toml

@@ -9,4 +9,4 @@ edition = "2021"
 penumbra-storage = { path = "../../storage" }
 async-trait = "0.1.52"
 anyhow = "1"
-tendermint = "0.29.0"
+tendermint = "0.31.1"

crates/component/src/component.rs

@@ -4,7 +4,7 @@ use anyhow::Result;
 use async_trait::async_trait;
 
 use penumbra_storage::StateWrite;
-use tendermint::abci;
+use tendermint::v0_34::abci;
 
 /// A component of the Penumbra application.
 #[async_trait]

crates/dao/Cargo.toml

@@ -29,8 +29,8 @@ prost = "0.11"
 serde = { version = "1", features = ["derive"] }
 metrics = "0.19.0"
 pbjson-types = "0.5.1"
-tendermint = "0.29.0"
-tendermint-light-client-verifier = "0.29.0"
+tendermint = "0.31.1"
+tendermint-light-client-verifier = "0.31.1"
 sha2 = "0.10.6"
 once_cell = "1.17.1"
 base64 = "0.20"

crates/distributions/Cargo.toml

@@ -24,6 +24,6 @@ penumbra-shielded-pool = { path = "../shielded-pool", default-features = false }
 async-trait = "0.1.52"
 anyhow = "1"
 tracing = "0.1"
-tendermint = "0.29.0"
+tendermint = "0.31.1"
 
 [dev-dependencies]

crates/ibc/Cargo.toml

@@ -20,8 +20,12 @@ penumbra-crypto = { path = "../../crypto", default-features = false }
 penumbra-shielded-pool = { path = "../shielded-pool", default-features = false }
 
 # Penumbra dependencies
-ibc-types = { git = "https://github.com/penumbra-zone/ibc-types", branch = "0.1.x",  version = "0.1.1" }
-ibc-proto = { version = "0.26", default-features = false }
+ibc-types = { git = "https://github.com/penumbra-zone/ibc-types", branch = "0.2.x"}
+ibc-proto = { git = "https://github.com/penumbra-zone/ibc-proto-rs", branch = "penumbra"}
+
+# Tendermint deps 
+tendermint = "0.31.1"
+tendermint-light-client-verifier = "0.31.1"
 
 # Crates.io deps
 ark-ff = { version = "0.4", default_features = false }
@@ -33,12 +37,10 @@ prost = "0.11"
 serde = { version = "1", features = ["derive"] }
 metrics = "0.19.0"
 pbjson-types = "0.5.1"
-tendermint = "0.29.0"
-tendermint-light-client-verifier = "0.29.0"
 sha2 = "0.10.6"
 once_cell = "1.17.1"
 base64 = "0.20"
 blake2b_simd = "0.5"
 
 [dev-dependencies]
-tokio = {version = "1.3", features = ["full"]}
+tokio = {version = "1.3", features = ["full"]}

crates/ibc/src/component/ibc_component.rs

@@ -8,7 +8,7 @@ use ibc_types::{
 };
 use penumbra_component::Component;
 use penumbra_storage::StateWrite;
-use tendermint::abci;
+use tendermint::v0_34::abci;
 use tracing::instrument;
 
 use crate::component::{client::StateWriteExt as _, client_counter::ClientCounter};

crates/ibc/src/ibc_action.rs

@@ -205,63 +205,63 @@ impl From<IbcAction> for pb::IbcAction {
         let raw_action = match value {
             IbcAction::CreateClient(msg) => pbjson_types::Any {
                 type_url: CREATE_CLIENT.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::UpdateClient(msg) => pbjson_types::Any {
                 type_url: UPDATE_CLIENT.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::ConnectionOpenInit(msg) => pbjson_types::Any {
                 type_url: CONNECTION_OPEN_INIT.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::ConnectionOpenTry(msg) => pbjson_types::Any {
                 type_url: CONNECTION_OPEN_TRY.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::ConnectionOpenAck(msg) => pbjson_types::Any {
                 type_url: CONNECTION_OPEN_ACK.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::ConnectionOpenConfirm(msg) => pbjson_types::Any {
                 type_url: CONNECTION_OPEN_CONFIRM.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::Acknowledgement(msg) => pbjson_types::Any {
                 type_url: ACKNOWLEDGEMENT.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::ChannelOpenInit(msg) => pbjson_types::Any {
                 type_url: CHANNEL_OPEN_INIT.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::ChannelOpenTry(msg) => pbjson_types::Any {
                 type_url: CHANNEL_OPEN_TRY.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::ChannelOpenAck(msg) => pbjson_types::Any {
                 type_url: CHANNEL_OPEN_ACK.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::ChannelOpenConfirm(msg) => pbjson_types::Any {
                 type_url: CHANNEL_OPEN_CONFIRM.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::ChannelCloseInit(msg) => pbjson_types::Any {
                 type_url: CHANNEL_CLOSE_INIT.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::ChannelCloseConfirm(msg) => pbjson_types::Any {
                 type_url: CHANNEL_CLOSE_CONFIRM.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::RecvPacket(msg) => pbjson_types::Any {
                 type_url: RECV_PACKET.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::Timeout(msg) => pbjson_types::Any {
                 type_url: TIMEOUT.to_string(),
-                value: msg.encode_vec().unwrap().into(),
+                value: msg.encode_vec().into(),
             },
             IbcAction::Unknown(raw_action) => raw_action,
         };

crates/sct/Cargo.toml

@@ -31,7 +31,7 @@ serde = { version = "1", features = ["derive"] }
 tracing = "0.1"
 anyhow = "1"
 async-trait = "0.1.52"
-tendermint = "0.29.0"
+tendermint = "0.31.1"
 blake2b_simd = "0.5"
 bytes = "1"
 rand_core = { version = "0.6.3", features = ["getrandom"] }

crates/shielded-pool/Cargo.toml

@@ -40,7 +40,7 @@ serde = { version = "1", features = ["derive"] }
 tracing = "0.1"
 anyhow = "1"
 async-trait = "0.1.52"
-tendermint = "0.29.0"
+tendermint = "0.31.1"
 blake2b_simd = "0.5"
 bytes = "1"
 rand_core = { version = "0.6.3", features = ["getrandom"] }

crates/shielded-pool/src/component/shielded_pool.rs

@@ -10,7 +10,7 @@ use penumbra_crypto::{asset, Value};
 use penumbra_proto::StateReadProto;
 use penumbra_storage::StateRead;
 use penumbra_storage::StateWrite;
-use tendermint::abci;
+use tendermint::v0_34::abci;
 
 use crate::state_key;
 

crates/stake/Cargo.toml

@@ -38,7 +38,7 @@ tokio = { version = "1.21.1", features = ["full", "tracing"], optional = true }
 im = { version = "15.1.0", optional = true }
 futures = { version = "0.3", optional = true }
 async-trait = { version = "0.1.52" }
-tendermint = { version = "0.29.0" }
+tendermint = { version = "0.31.1" }
 anyhow = "1"
 tracing = "0.1"
 serde = { version = "1", features = ["derive"] }

crates/stake/src/component.rs

@@ -28,14 +28,11 @@ use penumbra_proto::{
 use penumbra_shielded_pool::component::{NoteManager, SupplyRead, SupplyWrite};
 use penumbra_storage::{StateRead, StateWrite};
 use sha2::{Digest, Sha256};
+use tendermint::v0_34::abci;
 use tendermint::validator::Update;
-use tendermint::{
-    abci::{
-        self,
-        types::{Evidence, LastCommitInfo},
-    },
-    block, PublicKey,
-};
+
+use tendermint::abci::types::Misbehavior;
+use tendermint::{abci::types::CommitInfo, block, PublicKey};
 use tokio::task::JoinSet;
 use tracing::{instrument, Instrument};
 
@@ -644,7 +641,7 @@ pub(crate) trait StakingImpl: StateWriteExt {
     }
 
     #[instrument(skip(self, last_commit_info))]
-    async fn track_uptime(&mut self, last_commit_info: &LastCommitInfo) -> Result<()> {
+    async fn track_uptime(&mut self, last_commit_info: &CommitInfo) -> Result<()> {
         // Note: this probably isn't the correct height for the LastCommitInfo,
         // which is about the *last* commit, but at least it'll be consistent,
         // which is all we need to count signatures.
@@ -859,9 +856,11 @@ pub(crate) trait StakingImpl: StateWriteExt {
         Ok(())
     }
 
-    async fn process_evidence(&mut self, evidence: &Evidence) -> Result<()> {
+    async fn process_misbehavior(&mut self, misbehavior: &Misbehavior) -> Result<()> {
+        // TODO(erwan): noticed during the tendermints-rs@0.31 upgrade that lightclient attack
+        // detection is WIP.
         let validator = self
-            .validator_by_tendermint_address(&evidence.validator.address)
+            .validator_by_tendermint_address(&misbehavior.validator.address)
             .await?
             .ok_or_else(|| anyhow::anyhow!("attempted to slash unknown validator"))?;
 
@@ -944,7 +943,7 @@ impl Component for Staking {
         // For each validator identified as byzantine by tendermint, update its
         // state to be slashed
         for evidence in begin_block.byzantine_validators.iter() {
-            state.process_evidence(evidence).await.unwrap();
+            state.process_misbehavior(evidence).await.unwrap();
         }
 
         state

narsil/Cargo.toml

@@ -21,7 +21,8 @@ penumbra-tower-trace = { path = "../tower-trace" }
 penumbra-tendermint-proxy = { path = "../tendermint-proxy" }
 
 # Penumbra dependencies
-tower-abci = "0.5.0"
+# TODO(erwan): switch with upstream MERGEBLOCK
+tower-abci = { git = "https://github.com/penumbra-zone/tower-abci", branch = "tendermint-31" }
 tower-actor = "0.1.0"
 
 # External dependencies
@@ -40,10 +41,10 @@ metrics-tracing-context = "0.11.0"
 metrics-util = "0.13"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
-tendermint-config = "0.29.0"
-tendermint-proto = "0.29.0"
-tendermint = "0.29.0"
-tendermint-light-client-verifier = "0.29.0"
+tendermint-config = "0.31.1"
+tendermint-proto = "0.31.1"
+tendermint = "0.31.1"
+tendermint-light-client-verifier = "0.31.1"
 tokio = { version = "1.22", features = ["full"] }
 tokio-stream = "0.1"
 tokio-util = "0.7"
@@ -52,7 +53,7 @@ tonic-web = "0.4.0"
 tower = { version = "0.4", features = ["full"] }
 tower-service = "0.3.2"
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "ansi"] }
+tracing-subscriber = { version = "0.3.17", features = ["env-filter", "ansi"] }
 url = "2"
 atty = "0.2"
 

narsil/src/bin/narsild.rs

@@ -160,7 +160,7 @@ async fn main() -> anyhow::Result<()> {
             let abci_server = tokio::task::Builder::new()
                 .name("abci_server")
                 .spawn(
-                    tower_abci::Server::builder()
+                    tower_abci::v034::Server::builder()
                         .consensus(consensus)
                         .snapshot(snapshot)
                         .mempool(mempool)

narsil/src/ledger/consensus.rs

@@ -78,6 +78,12 @@ impl Consensus {
                         .await
                         .expect("commit must succeed"),
                 ),
+                Request::PrepareProposal(_) => {
+                    unimplemented!("PrepareProposal unimplemented in Tendermint v0.34")
+                }
+                Request::ProcessProposal(_) => {
+                    unimplemented!("ProcessProposal unimplemented in Tendermint v0.34")
+                }
             }));
         }
         Ok(())