fix: off-by-one in sats number resolution

Author: Ludo Galabru

components/chainhook-cli/src/cli/mod.rs

@@ -13,8 +13,9 @@ use chainhook_event_observer::chainhooks::types::{
     StacksPrintEventBasedPredicate,
 };
 use chainhook_event_observer::hord::db::{
-    fetch_and_cache_blocks_in_hord_db, find_inscriptions_at_wached_outpoint,
-    find_latest_compacted_block_known, open_readonly_hord_db_conn, open_readwrite_hord_db_conn,
+    delete_data_in_hord_db, fetch_and_cache_blocks_in_hord_db,
+    find_inscriptions_at_wached_outpoint, find_latest_compacted_block_known,
+    open_readonly_hord_db_conn, open_readwrite_hord_db_conn,
     retrieve_satoshi_point_using_local_storage,
 };
 use chainhook_event_observer::observer::BitcoinConfig;
@@ -185,6 +186,9 @@ enum DbCommand {
     /// Update hord db
     #[clap(name = "update", bin_name = "update")]
     Update(UpdateHordDbCommand),
+    /// Rebuild inscriptions entries for a given block
+    #[clap(name = "drop", bin_name = "drop")]
+    Drop(DropHordDbCommand),
 }
 
 #[derive(Subcommand, PartialEq, Clone, Debug)]
@@ -265,6 +269,17 @@ struct UpdateHordDbCommand {
     pub config_path: Option<String>,
 }
 
+#[derive(Parser, PartialEq, Clone, Debug)]
+struct DropHordDbCommand {
+    /// Starting block
+    pub start_block: u64,
+    /// Starting block
+    pub end_block: u64,
+    /// Load config file path
+    #[clap(long = "config-path")]
+    pub config_path: Option<String>,
+}
+
 pub fn main() {
     let logger = hiro_system_kit::log::setup_logger();
     let _guard = hiro_system_kit::log::setup_global_logger(logger.clone());
@@ -454,11 +469,6 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
                 let config =
                     Config::default(cmd.devnet, cmd.testnet, cmd.mainnet, &cmd.config_path)?;
 
-                info!(
-                    ctx.expect_logger(),
-                    "Computing satoshi number for satoshi at offet 0 in 1st output of transaction {} (block ${})", cmd.txid, cmd.block_height
-                );
-
                 let hord_db_conn =
                     open_readonly_hord_db_conn(&config.expected_cache_path(), &ctx).unwrap();
 
@@ -536,6 +546,17 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
                 )
                 .await?;
             }
+            DbCommand::Drop(cmd) => {
+                let config = Config::default(false, false, false, &cmd.config_path)?;
+                let rw_hord_db_conn =
+                    open_readwrite_hord_db_conn(&config.expected_cache_path(), &ctx)?;
+                delete_data_in_hord_db(cmd.start_block, cmd.end_block, &rw_hord_db_conn, &ctx)?;
+                info!(
+                    ctx.expect_logger(),
+                    "Cleaning hord_db: {} blocks dropped",
+                    cmd.end_block - cmd.start_block + 1
+                );
+            }
         },
     }
     Ok(())
@@ -550,7 +571,7 @@ pub async fn perform_hord_db_update(
 ) -> Result<(), String> {
     info!(
         ctx.expect_logger(),
-        "Syncing hord_db: {} blocks to download ({start_block}: {end_block}), using {network_threads} network threads", end_block - start_block
+        "Syncing hord_db: {} blocks to download ({start_block}: {end_block}), using {network_threads} network threads", end_block - start_block + 1
     );
 
     let bitcoin_config = BitcoinConfig {

components/chainhook-event-observer/src/hord/db/mod.rs

@@ -305,17 +305,17 @@ pub fn update_transfered_inscription(
 pub fn find_latest_inscription_block_height(
     hord_db_conn: &Connection,
     _ctx: &Context,
-) -> Result<u64, String> {
+) -> Result<Option<u64>, String> {
     let args: &[&dyn ToSql] = &[];
     let mut stmt = hord_db_conn
         .prepare("SELECT block_height FROM inscriptions ORDER BY block_height DESC LIMIT 1")
         .unwrap();
     let mut rows = stmt.query(args).unwrap();
     while let Ok(Some(row)) = rows.next() {
         let block_height: u64 = row.get(0).unwrap();
-        return Ok(block_height);
+        return Ok(Some(block_height));
     }
-    Ok(0)
+    Ok(None)
 }
 
 pub fn find_latest_inscription_number(
@@ -418,6 +418,34 @@ pub fn remove_entry_from_blocks(block_id: u32, hord_db_conn: &Connection, ctx: &
     }
 }
 
+pub fn delete_blocks_in_block_range(
+    start_block: u32,
+    end_block: u32,
+    rw_hord_db_conn: &Connection,
+    ctx: &Context,
+) {
+    if let Err(e) = rw_hord_db_conn.execute(
+        "DELETE FROM blocks WHERE id >= ?1 AND id <= ?2",
+        rusqlite::params![&start_block, &end_block],
+    ) {
+        ctx.try_log(|logger| slog::error!(logger, "{}", e.to_string()));
+    }
+}
+
+pub fn delete_inscriptions_in_block_range(
+    start_block: u32,
+    end_block: u32,
+    rw_hord_db_conn: &Connection,
+    ctx: &Context,
+) {
+    if let Err(e) = rw_hord_db_conn.execute(
+        "DELETE FROM inscriptions WHERE block_height >= ?1 AND block_height <= ?2",
+        rusqlite::params![&start_block, &end_block],
+    ) {
+        ctx.try_log(|logger| slog::error!(logger, "{}", e.to_string()));
+    }
+}
+
 pub fn remove_entry_from_inscriptions(
     inscription_id: &str,
     hord_db_conn: &Connection,
@@ -513,6 +541,17 @@ pub fn remove_entry_from_inscriptions(
 //     Ok(())
 // }
 
+pub fn delete_data_in_hord_db(
+    start_block: u64,
+    end_block: u64,
+    rw_hord_db_conn: &Connection,
+    ctx: &Context,
+) -> Result<(), String> {
+    delete_blocks_in_block_range(start_block as u32, end_block as u32, rw_hord_db_conn, &ctx);
+    delete_inscriptions_in_block_range(start_block as u32, end_block as u32, rw_hord_db_conn, &ctx);
+    Ok(())
+}
+
 pub async fn fetch_and_cache_blocks_in_hord_db(
     bitcoin_config: &BitcoinConfig,
     rw_hord_db_conn: &Connection,
@@ -521,6 +560,7 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
     ctx: &Context,
     network_thread: usize,
 ) -> Result<(), String> {
+    let number_of_blocks_to_process = end_block - start_block + 1;
     let retrieve_block_hash_pool = ThreadPool::new(network_thread);
     let (block_hash_tx, block_hash_rx) = crossbeam_channel::unbounded();
     let retrieve_block_data_pool = ThreadPool::new(network_thread);
@@ -589,29 +629,32 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
         .expect("unable to spawn thread");
 
     let mut blocks_stored = 0;
-    let mut cursor = find_latest_inscription_block_height(&rw_hord_db_conn, &ctx)
+    let mut cursor = 1 + find_latest_inscription_block_height(&rw_hord_db_conn, &ctx)?
         .unwrap_or(first_inscription_block_height) as usize;
     let mut inbox = HashMap::new();
 
     while let Ok(Some((block_height, compacted_block, raw_block))) = block_compressed_rx.recv() {
         ctx.try_log(|logger| slog::info!(logger, "Storing compacted block #{block_height}"));
 
         insert_entry_in_blocks(block_height, &compacted_block, &rw_hord_db_conn, &ctx);
+        blocks_stored += 1;
 
+        // println!("{} < {}", raw_block.height, cursor);
         // Early return, only considering blocks after 1st inscription
-        if raw_block.height < cursor {
-            continue;
-        }
-        let block_height = raw_block.height;
+        // if raw_block.height < cursor {
+        //     continue;
+        // }
+
+        // let block_height = raw_block.height;
         inbox.insert(raw_block.height, raw_block);
 
         // In the context of ordinals, we're constrained to process blocks sequentially
         // Blocks are processed by a threadpool and could be coming out of order.
         // Inbox block for later if the current block is not the one we should be
         // processing.
-        if block_height != cursor {
-            continue;
-        }
+        // if block_height != cursor {
+        //     continue;
+        // }
 
         // Is the action of processing a block allows us
         // to process more blocks present in the inbox?
@@ -627,9 +670,12 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
                 }
             };
 
-            if let Err(e) =
-                update_hord_db_and_augment_bitcoin_block(&mut new_block, &rw_hord_db_conn, &ctx)
-            {
+            if let Err(e) = update_hord_db_and_augment_bitcoin_block(
+                &mut new_block,
+                &rw_hord_db_conn,
+                &ctx,
+                false,
+            ) {
                 ctx.try_log(|logger| {
                     slog::error!(logger, "Unable to augment bitcoin block with hord_db: {e}",)
                 });
@@ -638,8 +684,7 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
             cursor += 1;
         }
 
-        blocks_stored += 1;
-        if blocks_stored == end_block - start_block {
+        if blocks_stored == number_of_blocks_to_process {
             let _ = block_data_tx.send(None);
             let _ = block_hash_tx.send(None);
             ctx.try_log(|logger| {
@@ -663,6 +708,15 @@ pub fn retrieve_satoshi_point_using_local_storage(
     transaction_identifier: &TransactionIdentifier,
     ctx: &Context,
 ) -> Result<(u64, u64, u64, u32), String> {
+    ctx.try_log(|logger| {
+        slog::info!(
+            logger,
+            "Computing Satoshi # for sat_point {}:0:0 (block #{})",
+            transaction_identifier.hash,
+            block_identifier.index
+        )
+    });
+
     let mut ordinal_offset = 0;
     let mut ordinal_block_number = block_identifier.index as u32;
     let txid = {
@@ -676,7 +730,7 @@ pub fn retrieve_satoshi_point_using_local_storage(
         let res = match find_compacted_block_at_block_height(ordinal_block_number, &hord_db_conn) {
             Some(res) => res,
             None => {
-                return Err(format!("unable to retrieve block ##{ordinal_block_number}"));
+                return Err(format!("unable to retrieve block #{ordinal_block_number}"));
             }
         };
 

components/chainhook-event-observer/src/hord/mod.rs

@@ -65,8 +65,9 @@ pub fn update_hord_db_and_augment_bitcoin_block(
     new_block: &mut BitcoinBlockData,
     rw_hord_db_conn: &Connection,
     ctx: &Context,
+    write_block: bool,
 ) -> Result<(), String> {
-    {
+    if write_block {
         ctx.try_log(|logger| {
             slog::info!(
                 logger,
@@ -156,9 +157,10 @@ pub fn update_hord_db_and_augment_bitcoin_block(
                     ctx.try_log(|logger| {
                         slog::info!(
                             logger,
-                            "Transaction {} in block {} includes a new inscription {}",
+                            "Transaction {} in block {} inscribed some content ({}) on Satoshi #{}",
                             new_tx.transaction_identifier.hash,
                             new_block.block_identifier.index,
+                            inscription.content_type,
                             ordinal_number
                         );
                     });
@@ -177,7 +179,7 @@ pub fn update_hord_db_and_augment_bitcoin_block(
 
         // Have inscriptions been transfered?
         let mut sats_in_offset = 0;
-        let mut sats_out_offset = 0;
+        let mut sats_out_offset = new_tx.metadata.outputs[0].value;
 
         for input in new_tx.metadata.inputs.iter() {
             // input.previous_output.txid
@@ -192,39 +194,32 @@ pub fn update_hord_db_and_augment_bitcoin_block(
             let entries =
                 find_inscriptions_at_wached_outpoint(&outpoint_pre_transfer, &rw_hord_db_conn);
 
-            ctx.try_log(|logger| {
-                slog::info!(
-                    logger,
-                    "Checking if {} is part of our watch outpoints set: {}",
-                    outpoint_pre_transfer,
-                    entries.len(),
-                )
-            });
+            // ctx.try_log(|logger| {
+            //     slog::info!(
+            //         logger,
+            //         "Checking if {} is part of our watch outpoints set: {}",
+            //         outpoint_pre_transfer,
+            //         entries.len(),
+            //     )
+            // });
 
             for (inscription_id, inscription_number, ordinal_number, offset) in entries.into_iter()
             {
                 let satpoint_pre_transfer = format!("{}:{}", outpoint_pre_transfer, offset);
-                // At this point we know that inscriptions are being moved.
-                ctx.try_log(|logger| {
-                    slog::info!(
-                        logger,
-                        "Detected transaction {} involving txin {} that includes watched ordinals",
-                        new_tx.transaction_identifier.hash,
-                        satpoint_pre_transfer,
-                    )
-                });
 
                 // Question is: are inscriptions moving to a new output,
                 // burnt or lost in fees and transfered to the miner?
                 let post_transfer_output = loop {
                     if sats_out_offset >= sats_in_offset + offset {
                         break Some(post_transfer_output_index);
                     }
-                    if post_transfer_output_index >= new_tx.metadata.outputs.len() {
+                    if post_transfer_output_index + 1 >= new_tx.metadata.outputs.len() {
                         break None;
+                    } else {
+                        post_transfer_output_index += 1;
+                        sats_out_offset +=
+                            new_tx.metadata.outputs[post_transfer_output_index].value;
                     }
-                    sats_out_offset += new_tx.metadata.outputs[post_transfer_output_index].value;
-                    post_transfer_output_index += 1;
                 };
 
                 let (outpoint_post_transfer, offset_post_transfer, updated_address) =
@@ -271,12 +266,24 @@ pub fn update_hord_db_and_augment_bitcoin_block(
                         }
                     };
 
+                // ctx.try_log(|logger| {
+                //     slog::info!(
+                //         logger,
+                //         "Updating watched outpoint {} to outpoint {}",
+                //         outpoint_post_transfer,
+                //         outpoint_pre_transfer,
+                //     )
+                // });
+                // At this point we know that inscriptions are being moved.
                 ctx.try_log(|logger| {
                     slog::info!(
                         logger,
-                        "Updating watched outpoint {} to outpoint {}",
+                        "Transaction {} in block {} moved inscribed Satoshi #{} from {} to {}",
+                        new_tx.transaction_identifier.hash,
+                        new_block.block_identifier.index,
+                        ordinal_number,
+                        satpoint_pre_transfer,
                         outpoint_post_transfer,
-                        outpoint_pre_transfer,
                     )
                 });
 

components/chainhook-event-observer/src/observer/mod.rs

@@ -554,6 +554,7 @@ pub async fn start_observer_commands_handler(
                                         block,
                                         &rw_hord_db_conn,
                                         &ctx,
+                                        true,
                                     ) {
                                         ctx.try_log(|logger| {
                                             slog::error!(
@@ -663,6 +664,7 @@ pub async fn start_observer_commands_handler(
                                         block,
                                         &rw_hord_db_conn,
                                         &ctx,
+                                        true,
                                     ) {
                                         ctx.try_log(|logger| {
                                             slog::error!(