fix: network, cascade changes

Author: Ludo Galabru

Cargo.lock

@@ -15,8 +15,8 @@ redis = "0.21.5"
 serde-redis = "0.12.0"
 # tikv-client = { git = "https://github.com/tikv/client-rust.git", rev = "8f54e6114227718e256027df2577bbacdf425f86" }
 # raft-proto = { git = "https://github.com/tikv/raft-rs", rev="f73766712a538c2f6eb135b455297ad6c03fc58d", version = "0.7.0"}
-chainhook-event-observer = { version = "=1.0.4", default-features = false, path = "../chainhook-event-observer" }
-chainhook-types = { version = "=1.0.2", path = "../chainhook-types-rs" }
+chainhook-event-observer = { version = "=1.0.4", default-features = false, features = ["ordinals"], path = "../chainhook-event-observer" }
+chainhook-types = { version = "=1.0.3", path = "../chainhook-types-rs" }
 clarinet-files = "1"
 hiro-system-kit = "0.1.0"
 # clarinet-files = { path = "../../../clarinet/components/clarinet-files" }

components/chainhook-cli/Cargo.toml

@@ -9,30 +9,6 @@ use std::{sync::mpsc::channel, thread};
 
 use super::DigestingCommand;
 
-#[derive(Debug, Deserialize)]
-pub struct Record {
-    pub id: u64,
-    pub created_at: String,
-    pub kind: RecordKind,
-    pub raw_log: String,
-}
-
-#[derive(Debug, Deserialize)]
-pub enum RecordKind {
-    #[serde(rename = "/new_block")]
-    StacksBlockReceived,
-    #[serde(rename = "/new_microblocks")]
-    StacksMicroblockReceived,
-    #[serde(rename = "/new_burn_block")]
-    BitcoinBlockReceived,
-    #[serde(rename = "/new_mempool_tx")]
-    TransactionAdmitted,
-    #[serde(rename = "/drop_mempool_tx")]
-    TransactionDropped,
-    #[serde(rename = "/attachments/new")]
-    AttachmentReceived,
-}
-
 pub fn start(
     digestion_tx: Sender<DigestingCommand>,
     config: &Config,

components/chainhook-cli/src/block/ingestion.rs

@@ -1,5 +1,5 @@
-pub mod digestion;
-pub mod ingestion;
+// pub mod digestion;
+// pub mod ingestion;
 use chainhook_types::BlockIdentifier;
 
 #[derive(Debug, Clone, Eq, PartialEq)]
@@ -9,3 +9,27 @@ pub enum DigestingCommand {
     Kill,
     Terminate,
 }
+
+#[derive(Debug, Deserialize)]
+pub struct Record {
+    pub id: u64,
+    pub created_at: String,
+    pub kind: RecordKind,
+    pub raw_log: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub enum RecordKind {
+    #[serde(rename = "/new_block")]
+    StacksBlockReceived,
+    #[serde(rename = "/new_microblocks")]
+    StacksMicroblockReceived,
+    #[serde(rename = "/new_burn_block")]
+    BitcoinBlockReceived,
+    #[serde(rename = "/new_mempool_tx")]
+    TransactionAdmitted,
+    #[serde(rename = "/drop_mempool_tx")]
+    TransactionDropped,
+    #[serde(rename = "/attachments/new")]
+    AttachmentReceived,
+}

components/chainhook-cli/src/block/mod.rs

@@ -91,6 +91,7 @@ impl Node {
             operators: HashSet::new(),
             display_logs: false,
             cache_path: self.config.storage.cache_path.clone(),
+            bitcoin_network: self.config.network.bitcoin_network.clone(),
         };
         info!(
             self.ctx.expect_logger(),

components/chainhook-cli/src/node/mod.rs

@@ -5,9 +5,14 @@ use chainhook_event_observer::chainhooks::bitcoin::{
     handle_bitcoin_hook_action, BitcoinChainhookOccurrence, BitcoinTriggerChainhook,
 };
 use chainhook_event_observer::chainhooks::types::BitcoinChainhookFullSpecification;
-use chainhook_event_observer::indexer;
+use chainhook_event_observer::indexer::ordinals::indexing::updater::OrdinalIndexUpdater;
+use chainhook_event_observer::indexer::ordinals::initialize_ordinal_index;
+use chainhook_event_observer::indexer::{self, BitcoinChainContext};
+use chainhook_event_observer::observer::{
+    EventObserverConfig, DEFAULT_CONTROL_PORT, DEFAULT_INGESTION_PORT,
+};
 use chainhook_event_observer::utils::{file_append, send_request, Context};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::time::Duration;
 
 pub async fn scan_bitcoin_chain_with_predicate(
@@ -133,8 +138,40 @@ pub async fn scan_bitcoin_chain_with_predicate(
             .result::<indexer::bitcoin::Block>()
             .map_err(|e| format!("unable to parse response ({})", e))?;
 
-        let block =
-            indexer::bitcoin::standardize_bitcoin_block(&config.network, cursor, raw_block, ctx)?;
+        let event_observer_config = EventObserverConfig {
+            normalization_enabled: true,
+            grpc_server_enabled: false,
+            hooks_enabled: true,
+            bitcoin_rpc_proxy_enabled: true,
+            event_handlers: vec![],
+            chainhook_config: None,
+            ingestion_port: DEFAULT_INGESTION_PORT,
+            control_port: DEFAULT_CONTROL_PORT,
+            bitcoin_node_username: config.network.bitcoin_node_rpc_username.clone(),
+            bitcoin_node_password: config.network.bitcoin_node_rpc_password.clone(),
+            bitcoin_node_rpc_url: config.network.bitcoin_node_rpc_url.clone(),
+            stacks_node_rpc_url: config.network.stacks_node_rpc_url.clone(),
+            operators: HashSet::new(),
+            display_logs: false,
+            cache_path: config.storage.cache_path.clone(),
+            bitcoin_network: config.network.bitcoin_network.clone(),
+        };
+
+        let ordinal_index = initialize_ordinal_index(&event_observer_config).unwrap();
+        match OrdinalIndexUpdater::update(&ordinal_index) {
+            Ok(_r) => {}
+            Err(e) => {}
+        }
+
+        let mut bitcoin_context = BitcoinChainContext::new(ordinal_index);
+
+        let block = indexer::bitcoin::standardize_bitcoin_block(
+            &config.network,
+            cursor,
+            raw_block,
+            &mut bitcoin_context,
+            ctx,
+        )?;
 
         let mut hits = vec![];
         for tx in block.transactions.iter() {

components/chainhook-cli/src/scan/bitcoin.rs

@@ -1,17 +1,19 @@
 use std::{
-    collections::{HashMap, VecDeque},
+    collections::{HashMap, HashSet, VecDeque},
     process,
 };
 
 use crate::{
     archive,
-    block::ingestion::{Record, RecordKind},
+    block::{Record, RecordKind},
     config::Config,
 };
 use chainhook_event_observer::{
     chainhooks::stacks::{
         handle_stacks_hook_action, StacksChainhookOccurrence, StacksTriggerChainhook,
     },
+    indexer::ordinals::{indexing::updater::OrdinalIndexUpdater, initialize_ordinal_index},
+    observer::{EventObserverConfig, DEFAULT_CONTROL_PORT, DEFAULT_INGESTION_PORT},
     utils::{file_append, send_request, AbstractStacksBlock},
 };
 use chainhook_event_observer::{
@@ -82,7 +84,28 @@ pub async fn scan_stacks_chain_with_predicate(
         let _ = record_tx.send(None);
     });
 
-    let mut indexer = Indexer::new(config.network.clone());
+    let event_observer_config = EventObserverConfig {
+        normalization_enabled: true,
+        grpc_server_enabled: false,
+        hooks_enabled: true,
+        bitcoin_rpc_proxy_enabled: true,
+        event_handlers: vec![],
+        chainhook_config: None,
+        ingestion_port: DEFAULT_INGESTION_PORT,
+        control_port: DEFAULT_CONTROL_PORT,
+        bitcoin_node_username: config.network.bitcoin_node_rpc_username.clone(),
+        bitcoin_node_password: config.network.bitcoin_node_rpc_password.clone(),
+        bitcoin_node_rpc_url: config.network.bitcoin_node_rpc_url.clone(),
+        stacks_node_rpc_url: config.network.stacks_node_rpc_url.clone(),
+        operators: HashSet::new(),
+        display_logs: false,
+        cache_path: config.storage.cache_path.clone(),
+        bitcoin_network: config.network.bitcoin_network.clone(),
+    };
+
+    let ordinal_index = initialize_ordinal_index(&event_observer_config).unwrap();
+
+    let mut indexer = Indexer::new(config.network.clone(), ordinal_index);
 
     let mut canonical_fork = {
         let mut cursor = BlockIdentifier::default();

components/chainhook-cli/src/scan/stacks.rs

@@ -41,6 +41,9 @@ crossbeam-channel = "0.5.6"
 redb = { version = "0.13.0", optional = true }
 chrono = { version = "0.4.19", optional = true }
 anyhow = { version = "1.0.56", features = ["backtrace"], optional = true  }
+futures = "0.3.21"
+hyper = { version = "0.14.24", features = ["http1", "client"] }
+hex = "0.4.3"
 
 [replace]
 "jsonrpc:0.13.0" = { git = 'https://github.com/apoelstra/rust-jsonrpc', rev = "1063671f122a8985c1b7c29030071253da515839" }

components/chainhook-event-observer/Cargo.toml

@@ -27,6 +27,7 @@ use rocket::serde::json::Value as JsonValue;
 use super::ordinals::indexing::updater::OrdinalIndexUpdater;
 use super::ordinals::indexing::OrdinalIndex;
 use super::ordinals::inscription::InscriptionParser;
+use super::ordinals::inscription_id::InscriptionId;
 use super::BitcoinChainContext;
 
 #[derive(Clone, PartialEq, Debug, Deserialize, Serialize)]
@@ -115,7 +116,15 @@ pub fn standardize_bitcoin_block(
     let mut transactions = vec![];
 
     match OrdinalIndexUpdater::update(&mut bitcoin_context.ordinal_index) {
-        Ok(_r) => {}
+        Ok(_) => {
+            ctx.try_log(|logger| {
+                slog::info!(
+                    logger,
+                    "Ordinal index updated (block count: {:?})",
+                    bitcoin_context.ordinal_index.block_count()
+                )
+            });
+        }
         Err(e) => {
             ctx.try_log(|logger| slog::error!(logger, "{}", e.to_string()));
         }
@@ -235,22 +244,15 @@ fn try_parse_ordinal_operation(
                     Err(_) => continue,
                 };
 
-                let outpoint = bitcoin::OutPoint {
+                let inscription_id = InscriptionId {
                     txid: tx.txid.clone(),
-                    vout: 0,
+                    index: 0,
                 };
-
                 let entries = ordinal_index.get_feed_inscriptions(3).unwrap();
                 ctx.try_log(|logger| slog::info!(logger, "Feed: {:?}", entries));
 
-                let inscription_ids = ordinal_index.get_inscriptions_on_output(outpoint).unwrap();
-                if inscription_ids.is_empty() {
-                    ctx.try_log(|logger| slog::info!(logger, "No inscriptions found in index, despite inscription detected in transaction"));
-                    return None;
-                }
-
                 let inscription_entry = match ordinal_index
-                    .get_inscription_entry(inscription_ids[0])
+                    .get_inscription_entry(inscription_id.clone())
                 {
                     Ok(Some(entry)) => entry,
                     _ => {
@@ -270,7 +272,7 @@ fn try_parse_ordinal_operation(
                                 .to_string(),
                             content_bytes: format!("0x{}", to_hex(&inscription_content_bytes)),
                             content_length: inscription_content_bytes.len(),
-                            inscription_id: "".into(),
+                            inscription_id: inscription_id.to_string(),
                             inscription_number: inscription_entry.number,
                             inscription_author: "".into(),
                             inscription_fee: inscription_entry.fee,

components/chainhook-event-observer/src/indexer/bitcoin/mod.rs

@@ -1,6 +1,6 @@
 use bitcoincore_rpc::bitcoin::Script;
 
-use crate::indexer::bitcoin::ordinal::InscriptionParser;
+use crate::indexer::ordinals::inscription::InscriptionParser;
 
 use super::super::tests::{helpers, process_bitcoin_blocks_and_check_expectations};
 

components/chainhook-event-observer/src/indexer/bitcoin/tests.rs

@@ -5,14 +5,16 @@ mod epoch;
 mod height;
 pub mod indexing;
 pub mod inscription;
-mod inscription_id;
+pub mod inscription_id;
 mod sat;
 mod sat_point;
 
 use std::time::Duration;
 
 type Result<T = (), E = anyhow::Error> = std::result::Result<T, E>;
 
+use chainhook_types::BitcoinNetwork;
+
 use crate::observer::EventObserverConfig;
 
 const DIFFCHANGE_INTERVAL: u64 =
@@ -24,12 +26,17 @@ const CYCLE_EPOCHS: u64 = 6;
 pub fn initialize_ordinal_index(
     config: &EventObserverConfig,
 ) -> Result<self::indexing::OrdinalIndex, String> {
+    let chain = match &config.bitcoin_network {
+        BitcoinNetwork::Mainnet => chain::Chain::Mainnet,
+        BitcoinNetwork::Testnet => chain::Chain::Testnet,
+        BitcoinNetwork::Regtest => chain::Chain::Regtest,
+    };
     let index_options = self::indexing::Options {
         rpc_username: config.bitcoin_node_username.clone(),
         rpc_password: config.bitcoin_node_password.clone(),
         data_dir: config.cache_path.clone().into(),
-        chain: chain::Chain::Mainnet,
-        first_inscription_height: None,
+        chain: chain,
+        first_inscription_height: Some(chain.first_inscription_height()),
         height_limit: None,
         index: None,
         rpc_url: config.bitcoin_node_rpc_url.clone(),

components/chainhook-event-observer/src/indexer/ordinals/mod.rs

@@ -147,6 +147,7 @@ impl EventObserverConfig {
             operators,
             display_logs: true,
             cache_path: config_file.cache_path.unwrap_or("cache".into()),
+            bitcoin_network: chainhook_types::BitcoinNetwork::Mainnet, // todo(lgalabru)
         };
         config
     }

components/chainhook-event-observer/src/main.rs

@@ -126,6 +126,7 @@ pub struct EventObserverConfig {
     pub operators: HashSet<String>,
     pub display_logs: bool,
     pub cache_path: String,
+    pub bitcoin_network: BitcoinNetwork,
 }
 
 #[derive(Deserialize, Debug)]
@@ -227,7 +228,7 @@ pub async fn start_event_observer(
     ctx.try_log(|logger| {
         slog::info!(
             logger,
-            "Initializing ordinals index in file {}",
+            "Initializing ordinals index in dir `{}`",
             config.cache_path
         )
     });

components/chainhook-event-observer/src/observer/mod.rs

@@ -40,6 +40,7 @@ fn generate_test_config() -> (EventObserverConfig, ChainhookStore) {
         operators,
         display_logs: false,
         cache_path: "cache".into(),
+        bitcoin_network: BitcoinNetwork::Regtest,
     };
     let mut entries = HashMap::new();
     entries.insert(ApiKey(None), ChainhookConfig::new());