fix: myriad of improvements

Author: Ludo Galabru

Cargo.lock

@@ -17,7 +17,7 @@ hex = "0.4.3"
 rand = "0.8.5"
 # tikv-client = { git = "https://github.com/tikv/client-rust.git", rev = "8f54e6114227718e256027df2577bbacdf425f86" }
 # raft-proto = { git = "https://github.com/tikv/raft-rs", rev="f73766712a538c2f6eb135b455297ad6c03fc58d", version = "0.7.0"}
-chainhook-sdk = { version = "0.2.0", default-features = false, features = ["ordinals", "zeromq"], path = "../chainhook-sdk" }
+chainhook-sdk = { version = "0.4.0", default-features = false, features = ["ordinals", "zeromq"], path = "../chainhook-sdk" }
 chainhook-types = { version = "1.0.6", path = "../chainhook-types-rs" }
 clarinet-files = "1"
 hiro-system-kit = "0.1.0"

components/chainhook-cli/Cargo.toml

@@ -859,7 +859,7 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
 
                     let mut missing_blocks = vec![];
                     for i in 1..=790000 {
-                        if find_lazy_block_at_block_height(i, 3, &blocks_db_rw).is_none() {
+                        if find_lazy_block_at_block_height(i, 10, &blocks_db_rw, &ctx).is_none() {
                             println!("Missing block {i}");
                             missing_blocks.push(i);
                         }

components/chainhook-cli/src/cli/mod.rs

@@ -80,7 +80,7 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate(
         let blocks_db_rw =
             open_readwrite_hord_db_conn_rocks_db(&config.expected_cache_path(), ctx)?;
 
-        if find_lazy_block_at_block_height(end_block as u32, 3, &blocks_db_rw).is_none() {
+        if find_lazy_block_at_block_height(end_block as u32, 10, &blocks_db_rw, &ctx).is_none() {
             // Count how many entries in the table
             // Compute the right interval
             // Start the build local storage routine
@@ -162,16 +162,13 @@ pub async fn scan_bitcoin_chainstate_via_rpc_using_predicate(
             }
 
             // Evaluating every single block is required for also keeping track of transfers.
-            let local_traverals = match find_all_inscriptions_in_block(
-                &cursor,
-                &inscriptions_db_conn,
-                &ctx,
-            )
-            .remove(&cursor)
-            {
-                Some(entry) => entry,
-                None => vec![],
-            };
+            let local_traverals =
+                match find_all_inscriptions_in_block(&cursor, &inscriptions_db_conn, &ctx)
+                    .remove(&cursor)
+                {
+                    Some(entry) => entry,
+                    None => vec![],
+                };
             for (transaction_identifier, traversal_result) in local_traverals.into_iter() {
                 traversals.insert(
                     (transaction_identifier, traversal_result.input_index),

components/chainhook-cli/src/scan/bitcoin.rs

@@ -119,7 +119,7 @@ impl Service {
         let context_cloned = self.ctx.clone();
         let event_observer_config_moved = event_observer_config.clone();
         let observer_command_tx_moved = observer_command_tx.clone();
-        let _ = std::thread::spawn(move || {
+        let _ = hiro_system_kit::thread_named("Chainhook event observer").spawn(move || {
             let future = start_event_observer(
                 event_observer_config_moved,
                 observer_command_tx_moved,

components/chainhook-cli/src/service/mod.rs

@@ -12,7 +12,7 @@ serde = "1"
 serde_json = { version = "1", features = ["arbitrary_precision"] }
 serde-hex = "0.1.0"
 serde_derive = "1"
-stacks-rpc-client = "=1.0.6"
+stacks-rpc-client = "=1.0.7"
 clarinet-utils = "1"
 hiro-system-kit = "0.1.0"
 # stacks-rpc-client = { version = "1", path = "../../../clarinet/components/stacks-rpc-client" }

components/chainhook-sdk/Cargo.toml

@@ -8,11 +8,11 @@ use chainhook_types::{
     BlockIdentifier, StacksChainEvent, StacksTransactionData, StacksTransactionEvent,
     StacksTransactionKind, TransactionIdentifier,
 };
-use stacks_rpc_client::clarity::stacks_common::codec::StacksMessageCodec;
-use stacks_rpc_client::clarity::vm::types::{CharType, SequenceData, Value as ClarityValue};
 use hiro_system_kit::slog;
 use reqwest::{Client, Method};
 use serde_json::Value as JsonValue;
+use stacks_rpc_client::clarity::stacks_common::codec::StacksMessageCodec;
+use stacks_rpc_client::clarity::vm::types::{CharType, SequenceData, Value as ClarityValue};
 use std::collections::{BTreeMap, HashMap};
 use std::io::Cursor;
 

components/chainhook-sdk/src/chainhooks/stacks/mod.rs

@@ -250,6 +250,7 @@ pub fn find_lazy_block_at_block_height(
     block_height: u32,
     retry: u8,
     blocks_db: &DB,
+    ctx: &Context,
 ) -> Option<LazyBlock> {
     let mut attempt = 0;
     // let mut read_options = rocksdb::ReadOptions::default();
@@ -265,6 +266,14 @@ pub fn find_lazy_block_at_block_height(
                 attempt += 1;
                 backoff = 2.0 * backoff + (backoff * rng.gen_range(0.0..1.0));
                 let duration = std::time::Duration::from_millis((backoff * 1_000.0) as u64);
+                ctx.try_log(|logger| {
+                    slog::warn!(
+                        logger,
+                        "Unable to find block {}, will retry in {:?}",
+                        block_height,
+                        duration
+                    )
+                });
                 std::thread::sleep(duration);
                 if attempt > retry {
                     return None;
@@ -496,9 +505,7 @@ pub fn find_all_inscriptions_in_block(
         let ordinal_number: u64 = row.get(1).unwrap();
         let block_height: u64 = row.get(2).unwrap();
         let inscription_id: String = row.get(3).unwrap();
-        let (transaction_identifier, input_index) = {
-            parse_inscription_id(&inscription_id)
-        };
+        let (transaction_identifier, input_index) = { parse_inscription_id(&inscription_id) };
         let inscription_offset_intra_output: u64 = row.get(4).unwrap();
         let outpoint_to_watch: String = row.get(5).unwrap();
         let (latest_transaction_id, mut output_index) = parse_outpoint_to_watch(&outpoint_to_watch);
@@ -508,7 +515,8 @@ pub fn find_all_inscriptions_in_block(
                 slog::warn!(
                     logger,
                     "Inscription {} ({}) most likely to end up lost in transfers",
-                    inscription_number, inscription_id
+                    inscription_number,
+                    inscription_id
                 )
             });
             output_index = 0;
@@ -961,31 +969,32 @@ pub fn retrieve_satoshi_point_using_lazy_storage(
     let mut ordinal_block_number = block_identifier.index as u32;
     let txid = transaction_identifier.get_8_hash_bytes();
 
-    let (sats_ranges, inscription_offset_cross_outputs) =
-        match traversals_cache.get(&(block_identifier.index as u32, txid.clone())) {
-            Some(entry) => {
-                let tx = entry.value();
-                (
-                    tx.get_sat_ranges(),
-                    tx.get_cumulated_sats_in_until_input_index(input_index),
-                )
+    let (sats_ranges, inscription_offset_cross_outputs) = match traversals_cache
+        .get(&(block_identifier.index as u32, txid.clone()))
+    {
+        Some(entry) => {
+            let tx = entry.value();
+            (
+                tx.get_sat_ranges(),
+                tx.get_cumulated_sats_in_until_input_index(input_index),
+            )
+        }
+        None => match find_lazy_block_at_block_height(ordinal_block_number, 10, &blocks_db, &ctx) {
+            None => {
+                return Err(format!("block #{ordinal_block_number} not in database"));
             }
-            None => match find_lazy_block_at_block_height(ordinal_block_number, 10, &blocks_db) {
-                None => {
-                    return Err(format!("block #{ordinal_block_number} not in database"));
+            Some(block) => match block.find_and_serialize_transaction_with_txid(&txid) {
+                Some(tx) => {
+                    let sats_ranges = tx.get_sat_ranges();
+                    let inscription_offset_cross_outputs =
+                        tx.get_cumulated_sats_in_until_input_index(input_index);
+                    traversals_cache.insert((ordinal_block_number, txid.clone()), tx);
+                    (sats_ranges, inscription_offset_cross_outputs)
                 }
-                Some(block) => match block.find_and_serialize_transaction_with_txid(&txid) {
-                    Some(tx) => {
-                        let sats_ranges = tx.get_sat_ranges();
-                        let inscription_offset_cross_outputs =
-                            tx.get_cumulated_sats_in_until_input_index(input_index);
-                        traversals_cache.insert((ordinal_block_number, txid.clone()), tx);
-                        (sats_ranges, inscription_offset_cross_outputs)
-                    }
-                    None => return Err(format!("txid not in block #{ordinal_block_number}")),
-                },
+                None => return Err(format!("txid not in block #{ordinal_block_number}")),
             },
-        };
+        },
+    };
 
     for (i, (min, max)) in sats_ranges.into_iter().enumerate() {
         if inscription_offset_cross_outputs >= min && inscription_offset_cross_outputs < max {
@@ -1068,13 +1077,13 @@ pub fn retrieve_satoshi_point_using_lazy_storage(
             }
         }
 
-        let lazy_block = match find_lazy_block_at_block_height(ordinal_block_number, 10, &blocks_db)
-        {
-            Some(block) => block,
-            None => {
-                return Err(format!("block #{ordinal_block_number} not in database"));
-            }
-        };
+        let lazy_block =
+            match find_lazy_block_at_block_height(ordinal_block_number, 10, &blocks_db, &ctx) {
+                Some(block) => block,
+                None => {
+                    return Err(format!("block #{ordinal_block_number} not in database"));
+                }
+            };
 
         let coinbase_txid = lazy_block.get_coinbase_txid();
         let txid = tx_cursor.0;

components/chainhook-sdk/src/hord/db/mod.rs

@@ -211,7 +211,6 @@ fn test_bitcoin_vector_040() {
 
 #[test]
 fn test_ordinal_inscription_parsing() {
-
     let bytes = hex::decode("208737bc46923c3e64c7e6768c0346879468bf3aba795a5f5f56efca288f50ed2aac0063036f7264010118746578742f706c61696e3b636861727365743d7574662d38004c9948656c6c6f2030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030300a68").unwrap();
 
     let script = Script::from(bytes);

components/chainhook-sdk/src/indexer/bitcoin/tests.rs

@@ -7,12 +7,12 @@ use crate::indexer::AssetClassCache;
 use crate::indexer::{IndexerConfig, StacksChainContext};
 use crate::utils::Context;
 use chainhook_types::*;
-use stacks_rpc_client::clarity::stacks_common::codec::StacksMessageCodec;
-use stacks_rpc_client::clarity::vm::types::{SequenceData, Value as ClarityValue};
-use stacks_rpc_client::clarity::codec::{StacksTransaction, TransactionAuth, TransactionPayload};
 use hiro_system_kit::slog;
 use rocket::serde::json::Value as JsonValue;
 use rocket::serde::Deserialize;
+use stacks_rpc_client::clarity::codec::{StacksTransaction, TransactionAuth, TransactionPayload};
+use stacks_rpc_client::clarity::stacks_common::codec::StacksMessageCodec;
+use stacks_rpc_client::clarity::vm::types::{SequenceData, Value as ClarityValue};
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::convert::TryInto;
 use std::io::Cursor;

components/chainhook-sdk/src/indexer/stacks/mod.rs

@@ -1049,7 +1049,9 @@ pub async fn start_observer_commands_handler(
                 });
 
                 if let Some(ref tx) = observer_events_tx {
-                    let _ = tx.send(ObserverEvent::PredicatesTriggered(chainhooks_to_trigger.len()));
+                    let _ = tx.send(ObserverEvent::PredicatesTriggered(
+                        chainhooks_to_trigger.len(),
+                    ));
                 }
                 for chainhook_to_trigger in chainhooks_to_trigger.into_iter() {
                     match handle_bitcoin_hook_action(chainhook_to_trigger, &proofs) {
@@ -1186,7 +1188,9 @@ pub async fn start_observer_commands_handler(
                 }
 
                 if let Some(ref tx) = observer_events_tx {
-                    let _ = tx.send(ObserverEvent::PredicatesTriggered(chainhooks_to_trigger.len()));
+                    let _ = tx.send(ObserverEvent::PredicatesTriggered(
+                        chainhooks_to_trigger.len(),
+                    ));
                 }
                 let proofs = HashMap::new();
                 for chainhook_to_trigger in chainhooks_to_trigger.into_iter() {