Namada rollback

.changelog/unreleased/features/1187-rollback.md

@@ -0,0 +1,2 @@
+- Added a rollback command to revert the Namada state to that of the previous
+  block. ([#1187](https://github.com/anoma/namada/pull/1187))

apps/src/bin/namada-node/cli.rs

@@ -40,6 +40,10 @@ pub fn main() -> Result<()> {
             cmds::Ledger::DumpDb(cmds::LedgerDumpDb(args)) => {
                 ledger::dump_db(ctx.config.ledger, args);
             }
+            cmds::Ledger::RollBack(_) => {
+                ledger::rollback(ctx.config.ledger)
+                    .wrap_err("Failed to rollback the Namada node")?;
+            }
         },
         cmds::NamadaNode::Config(sub) => match sub {
             cmds::Config::Gen(cmds::ConfigGen) => {

apps/src/lib/cli.rs

@@ -761,6 +761,7 @@ pub mod cmds {
         Run(LedgerRun),
         Reset(LedgerReset),
         DumpDb(LedgerDumpDb),
+        RollBack(LedgerRollBack),
     }
 
     impl SubCmd for Ledger {
@@ -771,8 +772,10 @@ pub mod cmds {
                 let run = SubCmd::parse(matches).map(Self::Run);
                 let reset = SubCmd::parse(matches).map(Self::Reset);
                 let dump_db = SubCmd::parse(matches).map(Self::DumpDb);
+                let rollback = SubCmd::parse(matches).map(Self::RollBack);
                 run.or(reset)
                     .or(dump_db)
+                    .or(rollback)
                     // The `run` command is the default if no sub-command given
                     .or(Some(Self::Run(LedgerRun(args::LedgerRun(None)))))
             })
@@ -787,6 +790,7 @@ pub mod cmds {
                 .subcommand(LedgerRun::def())
                 .subcommand(LedgerReset::def())
                 .subcommand(LedgerDumpDb::def())
+                .subcommand(LedgerRollBack::def())
         }
     }
 
@@ -846,6 +850,26 @@ pub mod cmds {
         }
     }
 
+    #[derive(Clone, Debug)]
+    pub struct LedgerRollBack;
+
+    impl SubCmd for LedgerRollBack {
+        const CMD: &'static str = "rollback";
+
+        fn parse(matches: &ArgMatches) -> Option<Self> {
+            matches.subcommand_matches(Self::CMD).map(|_matches| Self)
+        }
+
+        fn def() -> App {
+            App::new(Self::CMD).about(
+                "Roll Namada state back to the previous height. This command \
+                 does not create a backup of neither the Namada nor the \
+                 Tendermint state before execution: for extra safety, it is \
+                 recommended to make a backup in advance.",
+            )
+        }
+    }
+
     #[derive(Clone, Debug)]
     pub enum Config {
         Gen(ConfigGen),

apps/src/lib/node/ledger/mod.rs

@@ -217,6 +217,11 @@ pub fn dump_db(
     db.dump_last_block(out_file_path);
 }
 
+/// Roll Namada state back to the previous height
+pub fn rollback(config: config::Ledger) -> Result<(), shell::Error> {
+    shell::rollback(config)
+}
+
 /// Runs and monitors a few concurrent tasks.
 ///
 /// This includes:

apps/src/lib/node/ledger/shell/mod.rs

@@ -158,6 +158,22 @@ pub fn reset(config: config::Ledger) -> Result<()> {
     Ok(())
 }
 
+pub fn rollback(config: config::Ledger) -> Result<()> {
+    // Rollback Tendermint state
+    tracing::info!("Rollback Tendermint state");
+    let tendermint_block_height =
+        tendermint_node::rollback(config.tendermint_dir())
+            .map_err(Error::Tendermint)?;
+
+    // Rollback Namada state
+    let db_path = config.shell.db_dir(&config.chain_id);
+    let mut db = storage::PersistentDB::open(db_path, None);
+    tracing::info!("Rollback Namada state");
+
+    db.rollback(tendermint_block_height)
+        .map_err(|e| Error::StorageApi(storage_api::Error::new(e)))
+}
+
 #[derive(Debug)]
 #[allow(dead_code, clippy::large_enum_variant)]
 pub(super) enum ShellMode {

apps/src/lib/node/ledger/storage/rocksdb.rs

@@ -31,6 +31,7 @@ use std::cmp::Ordering;
 use std::fs::File;
 use std::path::Path;
 use std::str::FromStr;
+use std::sync::Mutex;
 
 use borsh::{BorshDeserialize, BorshSerialize};
 use data_encoding::HEXLOWER;
@@ -44,6 +45,7 @@ use namada::types::storage::{
     BlockHeight, BlockResults, Header, Key, KeySeg, KEY_SEGMENT_SEPARATOR,
 };
 use namada::types::time::DateTimeUtc;
+use rayon::prelude::*;
 use rocksdb::{
     BlockBasedOptions, Direction, FlushOptions, IteratorMode, Options,
     ReadOptions, SliceTransform, WriteBatch, WriteOptions,
@@ -302,6 +304,121 @@ impl RocksDB {
 
         println!("Done writing to {}", full_path.to_string_lossy());
     }
+
+    /// Rollback to previous block. Given the inner working of tendermint
+    /// rollback and of the key structure of Namada, calling rollback more than
+    /// once without restarting the chain results in a single rollback.
+    pub fn rollback(
+        &mut self,
+        tendermint_block_height: BlockHeight,
+    ) -> Result<()> {
+        let last_block = self.read_last_block()?.ok_or(Error::DBError(
+            "Missing last block in storage".to_string(),
+        ))?;
+        tracing::info!(
+            "Namada last block height: {}, Tendermint last block height: {}",
+            last_block.height,
+            tendermint_block_height
+        );
+
+        // If the block height to which tendermint rolled back matches the
+        // Namada height, there's no need to rollback
+        if tendermint_block_height == last_block.height {
+            tracing::info!(
+                "Namada height already matches the rollback Tendermint \
+                 height, no need to rollback."
+            );
+            return Ok(());
+        }
+
+        let mut batch = WriteBatch::default();
+        let previous_height =
+            BlockHeight::from(u64::from(last_block.height) - 1);
+
+        // Revert the non-height-prepended metadata storage keys which get
+        // updated with every block. Because of the way we save these
+        // three keys in storage we can only perform one rollback before
+        // restarting the chain
+        tracing::info!("Reverting non-height-prepended metadata keys");
+        batch.put("height", types::encode(&previous_height));
+        for metadata_key in [
+            "next_epoch_min_start_height",
+            "next_epoch_min_start_time",
+            "tx_queue",
+        ] {
+            let previous_key = format!("pred/{}", metadata_key);
+            let previous_value = self
+                .0
+                .get(previous_key.as_bytes())
+                .map_err(|e| Error::DBError(e.to_string()))?
+                .ok_or(Error::UnknownKey { key: previous_key })?;
+
+            batch.put(metadata_key, previous_value);
+            // NOTE: we cannot restore the "pred/" keys themselves since we
+            // don't have their predecessors in storage, but there's no need to
+            // since we cannot do more than one rollback anyway because of
+            // Tendermint.
+        }
+
+        // Delete block results for the last block
+        tracing::info!("Removing last block results");
+        batch.delete(format!("results/{}", last_block.height));
+
+        // Execute next step in parallel
+        let batch = Mutex::new(batch);
+
+        tracing::info!("Restoring previous hight subspace diffs");
+        self.iter_prefix(&Key::default())
+            .par_bridge()
+            .try_for_each(|(key, _value, _gas)| -> Result<()> {
+                // Restore previous height diff if present, otherwise delete the
+                // subspace key
+
+                // Add the prefix back since `iter_prefix` has removed it
+                let prefixed_key = format!("subspace/{}", key);
+
+                match self.read_subspace_val_with_height(
+                    &Key::from(key.to_db_key()),
+                    previous_height,
+                    last_block.height,
+                )? {
+                    Some(previous_value) => {
+                        batch.lock().unwrap().put(&prefixed_key, previous_value)
+                    }
+                    None => batch.lock().unwrap().delete(&prefixed_key),
+                }
+
+                Ok(())
+            })?;
+
+        // Delete any height-prepended key, including subspace diff keys
+        let mut batch = batch.into_inner().unwrap();
+        let prefix = last_block.height.to_string();
+        let mut read_opts = ReadOptions::default();
+        read_opts.set_total_order_seek(true);
+        let mut upper_prefix = prefix.clone().into_bytes();
+        if let Some(last) = upper_prefix.pop() {
+            upper_prefix.push(last + 1);
+        }
+        read_opts.set_iterate_upper_bound(upper_prefix);
+
+        let iter = self.0.iterator_opt(
+            IteratorMode::From(prefix.as_bytes(), Direction::Forward),
+            read_opts,
+        );
+        tracing::info!("Deleting keys prepended with the last height");
+        for (key, _value, _gas) in PersistentPrefixIterator(
+            // Empty prefix string to prevent stripping
+            PrefixIterator::new(iter, String::default()),
+        ) {
+            batch.delete(key);
+        }
+
+        // Write the batch and persist changes to disk
+        tracing::info!("Flushing restored state to disk");
+        self.exec_batch(batch)?;
+        self.flush(true)
+    }
 }
 
 impl DB for RocksDB {

apps/src/lib/node/ledger/tendermint_node.rs

@@ -6,6 +6,7 @@ use std::str::FromStr;
 use borsh::BorshSerialize;
 use namada::types::chain::ChainId;
 use namada::types::key::*;
+use namada::types::storage::BlockHeight;
 use namada::types::time::DateTimeUtc;
 use serde_json::json;
 #[cfg(feature = "abciplus")]
@@ -44,6 +45,8 @@ pub enum Error {
     StartUp(std::io::Error),
     #[error("{0}")]
     Runtime(String),
+    #[error("Failed to rollback tendermint state: {0}")]
+    RollBack(String),
     #[error("Failed to convert to String: {0:?}")]
     TendermintPath(std::ffi::OsString),
 }
@@ -190,6 +193,48 @@ pub fn reset(tendermint_dir: impl AsRef<Path>) -> Result<()> {
     Ok(())
 }
 
+pub fn rollback(tendermint_dir: impl AsRef<Path>) -> Result<BlockHeight> {
+    let tendermint_path = from_env_or_default()?;
+    let tendermint_dir = tendermint_dir.as_ref().to_string_lossy();
+
+    // Rollback tendermint state, see https://github.com/tendermint/tendermint/blob/main/cmd/tendermint/commands/rollback.go for details
+    // on how the tendermint rollback behaves
+    let output = std::process::Command::new(tendermint_path)
+        .args([
+            "rollback",
+            "unsafe-all",
+            // NOTE: log config: https://docs.tendermint.com/master/nodes/logging.html#configuring-log-levels
+            // "--log-level=\"*debug\"",
+            "--home",
+            &tendermint_dir,
+        ])
+        .output()
+        .map_err(|e| Error::RollBack(e.to_string()))?;
+
+    // Capture the block height from the output of tendermint rollback
+    // Tendermint stdout message: "Rolled
+    // back state to height %d and hash %v"
+    let output_msg = String::from_utf8(output.stdout)
+        .map_err(|e| Error::RollBack(e.to_string()))?;
+    let (_, right) = output_msg
+        .split_once("Rolled back state to height")
+        .ok_or(Error::RollBack(
+            "Missing expected block height in tendermint stdout message"
+                .to_string(),
+        ))?;
+
+    let mut sub = right.split_ascii_whitespace();
+    let height = sub.next().ok_or(Error::RollBack(
+        "Missing expected block height in tendermint stdout message"
+            .to_string(),
+    ))?;
+
+    Ok(height
+        .parse::<u64>()
+        .map_err(|e| Error::RollBack(e.to_string()))?
+        .into())
+}
+
 /// Convert a common signing scheme validator key into JSON for
 /// Tendermint
 fn validator_key_to_json(