Skip to content

Commit 3c1c99d

Browse files
author
Ludo Galabru
committed
feat: logic to start ingestion during indexing
1 parent 135297e commit 3c1c99d

File tree

5 files changed

+256
-156
lines changed

5 files changed

+256
-156
lines changed

‎components/chainhook-cli/src/cli/mod.rs

+17-122
Original file line numberDiff line numberDiff line change
@@ -10,22 +10,20 @@ use crate::storage::{
1010
open_readonly_stacks_db_conn,
1111
};
1212

13-
use chainhook_sdk::bitcoincore_rpc::{Auth, Client, RpcApi};
14-
1513
use chainhook_sdk::chainhooks::types::{
1614
BitcoinChainhookFullSpecification, BitcoinChainhookNetworkSpecification, BitcoinPredicateType, ChainhookFullSpecification, FileHook,
1715
HookAction, OrdinalOperations, StacksChainhookFullSpecification,
1816
StacksChainhookNetworkSpecification, StacksPredicate, StacksPrintEventBasedPredicate,
1917
};
2018
use chainhook_sdk::hord::db::{
21-
delete_data_in_hord_db, fetch_and_cache_blocks_in_hord_db, find_last_block_inserted,
22-
find_lazy_block_at_block_height, find_watched_satpoint_for_inscription, initialize_hord_db,
23-
open_readonly_hord_db_conn, open_readonly_hord_db_conn_rocks_db, open_readwrite_hord_db_conn,
19+
delete_data_in_hord_db, find_last_block_inserted, find_lazy_block_at_block_height,
20+
find_watched_satpoint_for_inscription, initialize_hord_db, open_readonly_hord_db_conn,
21+
open_readonly_hord_db_conn_rocks_db, open_readwrite_hord_db_conn,
2422
open_readwrite_hord_db_conn_rocks_db, retrieve_satoshi_point_using_lazy_storage,
2523
};
2624
use chainhook_sdk::hord::{
2725
new_traversals_lazy_cache, retrieve_inscribed_satoshi_points_from_block,
28-
update_storage_and_augment_bitcoin_block_with_inscription_transfer_data, HordConfig, Storage,
26+
update_storage_and_augment_bitcoin_block_with_inscription_transfer_data, Storage,
2927
};
3028
use chainhook_sdk::indexer;
3129
use chainhook_sdk::indexer::bitcoin::{
@@ -474,35 +472,6 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
474472

475473
info!(ctx.expect_logger(), "Starting service...",);
476474

477-
if !cmd.hord_disabled {
478-
info!(
479-
ctx.expect_logger(),
480-
"Ordinal indexing is enabled by default hord, checking index... (use --no-hord to disable ordinals)"
481-
);
482-
483-
if let Some((start_block, end_block)) = should_sync_hord_db(&config, &ctx)? {
484-
if start_block == 0 {
485-
info!(
486-
ctx.expect_logger(),
487-
"Initializing hord indexing from block #{}", start_block
488-
);
489-
} else {
490-
info!(
491-
ctx.expect_logger(),
492-
"Resuming hord indexing from block #{}", start_block
493-
);
494-
}
495-
perform_hord_db_update(
496-
start_block,
497-
end_block,
498-
&config.get_hord_config(),
499-
&config,
500-
&ctx,
501-
)
502-
.await?;
503-
}
504-
}
505-
506475
let mut service = Service::new(config, ctx);
507476
return service.run(predicates, cmd.hord_disabled).await;
508477
}
@@ -673,7 +642,8 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
673642
));
674643
}
675644
};
676-
645+
// TODO: if a stacks.rocksdb is present, use it.
646+
// TODO: update Stacks archive file if required.
677647
scan_stacks_chainstate_via_csv_using_predicate(
678648
&predicate_spec,
679649
&mut config,
@@ -731,11 +701,12 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
731701
find_last_block_inserted(&hord_db_conn) as u64
732702
};
733703
if cmd.block_height > tip_height {
734-
perform_hord_db_update(
704+
crate::hord::perform_hord_db_update(
735705
tip_height,
736706
cmd.block_height,
737707
&config.get_hord_config(),
738708
&config,
709+
None,
739710
&ctx,
740711
)
741712
.await?;
@@ -802,11 +773,12 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
802773
let tip_height = find_last_block_inserted(&blocks_db_conn) as u64;
803774
let _end_at = match cmd.block_height {
804775
Some(block_height) if block_height > tip_height => {
805-
perform_hord_db_update(
776+
crate::hord::perform_hord_db_update(
806777
tip_height,
807778
block_height,
808779
&config.get_hord_config(),
809780
&config,
781+
None,
810782
&ctx,
811783
)
812784
.await?;
@@ -860,7 +832,9 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
860832
Command::Hord(HordCommand::Db(subcmd)) => match subcmd {
861833
HordDbCommand::Sync(cmd) => {
862834
let config = Config::default(false, false, false, &cmd.config_path)?;
863-
if let Some((start_block, end_block)) = should_sync_hord_db(&config, &ctx)? {
835+
if let Some((start_block, end_block)) =
836+
crate::hord::should_sync_hord_db(&config, &ctx)?
837+
{
864838
if start_block == 0 {
865839
info!(
866840
ctx.expect_logger(),
@@ -872,11 +846,12 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
872846
"Resuming hord indexing from block #{}", start_block
873847
);
874848
}
875-
perform_hord_db_update(
849+
crate::hord::perform_hord_db_update(
876850
start_block,
877851
end_block,
878852
&config.get_hord_config(),
879853
&config,
854+
None,
880855
&ctx,
881856
)
882857
.await?;
@@ -902,11 +877,12 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
902877
)?;
903878
}
904879
// Update data
905-
perform_hord_db_update(
880+
crate::hord::perform_hord_db_update(
906881
cmd.start_block,
907882
cmd.end_block,
908883
&config.get_hord_config(),
909884
&config,
885+
None,
910886
&ctx,
911887
)
912888
.await?;
@@ -1023,87 +999,6 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
1023999
Ok(())
10241000
}
10251001

1026-
pub fn should_sync_hord_db(config: &Config, ctx: &Context) -> Result<Option<(u64, u64)>, String> {
1027-
let auth = Auth::UserPass(
1028-
config.network.bitcoind_rpc_username.clone(),
1029-
config.network.bitcoind_rpc_password.clone(),
1030-
);
1031-
1032-
let bitcoin_rpc = match Client::new(&config.network.bitcoind_rpc_url, auth) {
1033-
Ok(con) => con,
1034-
Err(message) => {
1035-
return Err(format!("Bitcoin RPC error: {}", message.to_string()));
1036-
}
1037-
};
1038-
1039-
let start_block = match open_readonly_hord_db_conn_rocks_db(&config.expected_cache_path(), &ctx)
1040-
{
1041-
Ok(blocks_db) => find_last_block_inserted(&blocks_db) as u64,
1042-
Err(err) => {
1043-
warn!(ctx.expect_logger(), "{}", err);
1044-
0
1045-
}
1046-
};
1047-
1048-
if start_block == 0 {
1049-
let _ = initialize_hord_db(&config.expected_cache_path(), &ctx);
1050-
}
1051-
1052-
let end_block = match bitcoin_rpc.get_blockchain_info() {
1053-
Ok(result) => result.blocks,
1054-
Err(e) => {
1055-
return Err(format!(
1056-
"unable to retrieve Bitcoin chain tip ({})",
1057-
e.to_string()
1058-
));
1059-
}
1060-
};
1061-
1062-
if start_block < end_block {
1063-
Ok(Some((start_block, end_block)))
1064-
} else {
1065-
Ok(None)
1066-
}
1067-
}
1068-
1069-
pub async fn perform_hord_db_update(
1070-
start_block: u64,
1071-
end_block: u64,
1072-
hord_config: &HordConfig,
1073-
config: &Config,
1074-
ctx: &Context,
1075-
) -> Result<(), String> {
1076-
info!(
1077-
ctx.expect_logger(),
1078-
"Syncing hord_db: {} blocks to download ({start_block}: {end_block})",
1079-
end_block - start_block + 1
1080-
);
1081-
1082-
let bitcoin_config = BitcoinConfig {
1083-
username: config.network.bitcoind_rpc_username.clone(),
1084-
password: config.network.bitcoind_rpc_password.clone(),
1085-
rpc_url: config.network.bitcoind_rpc_url.clone(),
1086-
network: config.network.bitcoin_network.clone(),
1087-
bitcoin_block_signaling: config.network.bitcoin_block_signaling.clone(),
1088-
};
1089-
1090-
let blocks_db = open_readwrite_hord_db_conn_rocks_db(&config.expected_cache_path(), &ctx)?;
1091-
let inscriptions_db_conn_rw = open_readwrite_hord_db_conn(&config.expected_cache_path(), &ctx)?;
1092-
1093-
let _ = fetch_and_cache_blocks_in_hord_db(
1094-
&bitcoin_config,
1095-
&blocks_db,
1096-
&inscriptions_db_conn_rw,
1097-
start_block,
1098-
end_block,
1099-
hord_config,
1100-
&ctx,
1101-
)
1102-
.await?;
1103-
1104-
Ok(())
1105-
}
1106-
11071002
#[allow(dead_code)]
11081003
pub fn install_ctrlc_handler(terminate_tx: Sender<DigestingCommand>, ctx: Context) {
11091004
ctrlc::set_handler(move || {

‎components/chainhook-cli/src/hord/mod.rs

+113
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
use std::sync::mpsc::Sender;
2+
3+
use chainhook_sdk::{
4+
bitcoincore_rpc::{Auth, Client, RpcApi},
5+
hord::{
6+
db::{
7+
fetch_and_cache_blocks_in_hord_db, find_last_block_inserted, initialize_hord_db,
8+
open_readonly_hord_db_conn_rocks_db, open_readwrite_hord_db_conn,
9+
open_readwrite_hord_db_conn_rocks_db, find_latest_inscription_block_height, open_readonly_hord_db_conn,
10+
},
11+
HordConfig,
12+
},
13+
observer::BitcoinConfig,
14+
utils::Context,
15+
};
16+
use chainhook_types::BitcoinBlockData;
17+
18+
use crate::config::Config;
19+
20+
pub fn should_sync_hord_db(config: &Config, ctx: &Context) -> Result<Option<(u64, u64)>, String> {
21+
let auth = Auth::UserPass(
22+
config.network.bitcoind_rpc_username.clone(),
23+
config.network.bitcoind_rpc_password.clone(),
24+
);
25+
26+
let bitcoin_rpc = match Client::new(&config.network.bitcoind_rpc_url, auth) {
27+
Ok(con) => con,
28+
Err(message) => {
29+
return Err(format!("Bitcoin RPC error: {}", message.to_string()));
30+
}
31+
};
32+
33+
let mut start_block = match open_readonly_hord_db_conn_rocks_db(&config.expected_cache_path(), &ctx)
34+
{
35+
Ok(blocks_db) => find_last_block_inserted(&blocks_db) as u64,
36+
Err(err) => {
37+
warn!(ctx.expect_logger(), "{}", err);
38+
0
39+
}
40+
};
41+
42+
if start_block == 0 {
43+
let _ = initialize_hord_db(&config.expected_cache_path(), &ctx);
44+
}
45+
46+
let inscriptions_db_conn = open_readonly_hord_db_conn(&config.expected_cache_path(), &ctx)?;
47+
48+
match find_latest_inscription_block_height(&inscriptions_db_conn, ctx)? {
49+
Some(height) => {
50+
start_block = start_block.min(height);
51+
}
52+
None => {
53+
start_block = start_block.min(config.get_hord_config().first_inscription_height);
54+
}
55+
};
56+
57+
58+
let end_block = match bitcoin_rpc.get_blockchain_info() {
59+
Ok(result) => result.blocks,
60+
Err(e) => {
61+
return Err(format!(
62+
"unable to retrieve Bitcoin chain tip ({})",
63+
e.to_string()
64+
));
65+
}
66+
};
67+
68+
if start_block < end_block {
69+
Ok(Some((start_block, end_block)))
70+
} else {
71+
Ok(None)
72+
}
73+
}
74+
75+
pub async fn perform_hord_db_update(
76+
start_block: u64,
77+
end_block: u64,
78+
hord_config: &HordConfig,
79+
config: &Config,
80+
block_post_processor: Option<Sender<BitcoinBlockData>>,
81+
ctx: &Context,
82+
) -> Result<(), String> {
83+
info!(
84+
ctx.expect_logger(),
85+
"Syncing hord_db: {} blocks to download ({start_block}: {end_block})",
86+
end_block - start_block + 1
87+
);
88+
89+
let bitcoin_config = BitcoinConfig {
90+
username: config.network.bitcoind_rpc_username.clone(),
91+
password: config.network.bitcoind_rpc_password.clone(),
92+
rpc_url: config.network.bitcoind_rpc_url.clone(),
93+
network: config.network.bitcoin_network.clone(),
94+
bitcoin_block_signaling: config.network.bitcoin_block_signaling.clone(),
95+
};
96+
97+
let blocks_db = open_readwrite_hord_db_conn_rocks_db(&config.expected_cache_path(), &ctx)?;
98+
let inscriptions_db_conn_rw = open_readwrite_hord_db_conn(&config.expected_cache_path(), &ctx)?;
99+
100+
let _ = fetch_and_cache_blocks_in_hord_db(
101+
&bitcoin_config,
102+
&blocks_db,
103+
&inscriptions_db_conn_rw,
104+
start_block,
105+
end_block,
106+
hord_config,
107+
block_post_processor,
108+
&ctx,
109+
)
110+
.await?;
111+
112+
Ok(())
113+
}

‎components/chainhook-cli/src/main.rs

+1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ pub mod archive;
1616
pub mod block;
1717
pub mod cli;
1818
pub mod config;
19+
pub mod hord;
1920
pub mod scan;
2021
pub mod service;
2122
pub mod storage;

0 commit comments

Comments
 (0)