Skip to content

Commit b0378c3

Browse files
author
Ludo Galabru
committed
feat: share traversals_cache over 10 blocks spans
1 parent 569d22f commit b0378c3

File tree

4 files changed

+78
-38
lines changed

4 files changed

+78
-38
lines changed

‎components/chainhook-cli/src/cli/mod.rs

+7-5
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ use chainhook_event_observer::hord::db::{
2323
open_readwrite_hord_db_conn_rocks_db, retrieve_satoshi_point_using_local_storage,
2424
};
2525
use chainhook_event_observer::hord::{
26-
retrieve_inscribed_satoshi_points_from_block,
26+
new_traversals_cache, retrieve_inscribed_satoshi_points_from_block,
2727
update_storage_and_augment_bitcoin_block_with_inscription_transfer_data, Storage,
2828
};
2929
use chainhook_event_observer::indexer;
@@ -633,14 +633,13 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
633633
};
634634

635635
let transaction_identifier = TransactionIdentifier { hash: txid.clone() };
636-
let hasher = FxBuildHasher::default();
637-
let cache = Arc::new(DashMap::with_hasher(hasher));
636+
let traversals_cache = new_traversals_cache();
638637
let traversal = retrieve_satoshi_point_using_local_storage(
639638
&hord_db_conn,
640639
&block_identifier,
641640
&transaction_identifier,
642641
0,
643-
cache,
642+
Arc::new(traversals_cache),
644643
&ctx,
645644
)?;
646645
info!(
@@ -655,10 +654,13 @@ async fn handle_command(opts: Opts, ctx: Context) -> Result<(), String> {
655654
let block =
656655
fetch_and_standardize_block(cmd.block_height, &bitcoin_config, &ctx)
657656
.await?;
658-
let traversals = retrieve_inscribed_satoshi_points_from_block(
657+
let traversals_cache = Arc::new(new_traversals_cache());
658+
659+
let _traversals = retrieve_inscribed_satoshi_points_from_block(
659660
&block,
660661
None,
661662
&config.expected_cache_path(),
663+
&traversals_cache,
662664
&ctx,
663665
);
664666
// info!(

‎components/chainhook-event-observer/src/hord/db/mod.rs

+11-3
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ use crate::{
2727
};
2828

2929
use super::{
30+
new_traversals_cache,
3031
ord::{height::Height, sat::Sat},
3132
update_hord_db_and_augment_bitcoin_block,
3233
};
@@ -955,6 +956,7 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
955956
let mut cursor = start_block as usize;
956957
let mut inbox = HashMap::new();
957958
let mut num_writes = 0;
959+
let traversals_cache = Arc::new(new_traversals_cache());
958960

959961
while let Ok(Some((block_height, compacted_block, raw_block))) = block_compressed_rx.recv() {
960962
insert_entry_in_blocks(block_height, &compacted_block, &blocks_db_rw, &ctx);
@@ -1002,6 +1004,7 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
10021004
&inscriptions_db_conn_rw,
10031005
false,
10041006
&hord_db_path,
1007+
&traversals_cache,
10051008
&ctx,
10061009
) {
10071010
ctx.try_log(|logger| {
@@ -1031,6 +1034,10 @@ pub async fn fetch_and_cache_blocks_in_hord_db(
10311034
return Ok(());
10321035
}
10331036

1037+
if num_writes % 10 == 0 {
1038+
traversals_cache.clear();
1039+
}
1040+
10341041
if num_writes % 5000 == 0 {
10351042
ctx.try_log(|logger| {
10361043
slog::info!(logger, "Flushing DB to disk ({num_writes} inserts)");
@@ -1079,7 +1086,7 @@ pub fn retrieve_satoshi_point_using_local_storage(
10791086
block_identifier: &BlockIdentifier,
10801087
transaction_identifier: &TransactionIdentifier,
10811088
inscription_number: u64,
1082-
cache: Arc<
1089+
traversals_cache: Arc<
10831090
DashMap<
10841091
(u32, [u8; 8]),
10851092
(Vec<([u8; 8], u32, u16, u64)>, Vec<u64>),
@@ -1116,7 +1123,7 @@ pub fn retrieve_satoshi_point_using_local_storage(
11161123
));
11171124
}
11181125

1119-
if let Some(cached_tx) = cache.get(&(ordinal_block_number, tx_cursor.0)) {
1126+
if let Some(cached_tx) = traversals_cache.get(&(ordinal_block_number, tx_cursor.0)) {
11201127
let (inputs, outputs) = cached_tx.value();
11211128
let mut next_found_in_cache = false;
11221129

@@ -1282,7 +1289,8 @@ pub fn retrieve_satoshi_point_using_local_storage(
12821289
// });
12831290

12841291
if sats_out < sats_in {
1285-
cache.insert((ordinal_block_number, txid_n), (inputs.clone(), outputs));
1292+
traversals_cache
1293+
.insert((ordinal_block_number, txid_n), (inputs.clone(), outputs));
12861294
ordinal_offset = sats_out - (sats_in - txin_value);
12871295
ordinal_block_number = *block_height;
12881296

‎components/chainhook-event-observer/src/hord/mod.rs

+26-5
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,14 @@ use chainhook_types::{
99
OrdinalOperation, TransactionIdentifier,
1010
};
1111
use dashmap::DashMap;
12+
use fxhash::{FxBuildHasher, FxHasher};
1213
use hiro_system_kit::slog;
1314
use rand::seq::SliceRandom;
1415
use rand::thread_rng;
1516
use rocksdb::DB;
1617
use rusqlite::Connection;
1718
use std::collections::{BTreeMap, HashMap, VecDeque};
19+
use std::hash::BuildHasherDefault;
1820
use std::path::PathBuf;
1921
use std::sync::mpsc::channel;
2022
use std::sync::Arc;
@@ -162,10 +164,24 @@ pub fn revert_hord_db_with_augmented_bitcoin_block(
162164
Ok(())
163165
}
164166

167+
pub fn new_traversals_cache(
168+
) -> DashMap<(u32, [u8; 8]), (Vec<([u8; 8], u32, u16, u64)>, Vec<u64>), BuildHasherDefault<FxHasher>>
169+
{
170+
let hasher = FxBuildHasher::default();
171+
DashMap::with_hasher(hasher)
172+
}
173+
165174
pub fn retrieve_inscribed_satoshi_points_from_block(
166175
block: &BitcoinBlockData,
167176
inscriptions_db_conn: Option<&Connection>,
168177
hord_db_path: &PathBuf,
178+
traversals_cache: &Arc<
179+
DashMap<
180+
(u32, [u8; 8]),
181+
(Vec<([u8; 8], u32, u16, u64)>, Vec<u64>),
182+
BuildHasherDefault<FxHasher>,
183+
>,
184+
>,
169185
ctx: &Context,
170186
) -> HashMap<TransactionIdentifier, TraversalResult> {
171187
let mut transactions_ids = vec![];
@@ -202,14 +218,12 @@ pub fn retrieve_inscribed_satoshi_points_from_block(
202218

203219
let mut rng = thread_rng();
204220
transactions_ids.shuffle(&mut rng);
205-
let hasher = fxhash::FxBuildHasher::default();
206-
let shared_cache = Arc::new(DashMap::with_hasher(hasher));
207221
for transaction_id in transactions_ids.into_iter() {
208222
let moved_traversal_tx = traversal_tx.clone();
209223
let moved_ctx = ctx.clone();
210224
let block_identifier = block.block_identifier.clone();
211225
let moved_hord_db_path = hord_db_path.clone();
212-
let cache = shared_cache.clone();
226+
let local_cache = traversals_cache.clone();
213227
traversal_data_pool.execute(move || loop {
214228
match open_readonly_hord_db_conn_rocks_db(&moved_hord_db_path, &moved_ctx) {
215229
Ok(blocks_db) => {
@@ -218,7 +232,7 @@ pub fn retrieve_inscribed_satoshi_points_from_block(
218232
&block_identifier,
219233
&transaction_id,
220234
0,
221-
cache,
235+
local_cache,
222236
&moved_ctx,
223237
);
224238
let _ = moved_traversal_tx.send((transaction_id, traversal));
@@ -262,7 +276,6 @@ pub fn retrieve_inscribed_satoshi_points_from_block(
262276
}
263277
}
264278
let _ = traversal_data_pool.join();
265-
std::thread::spawn(move || drop(shared_cache));
266279
}
267280

268281
traversals
@@ -274,6 +287,13 @@ pub fn update_hord_db_and_augment_bitcoin_block(
274287
inscriptions_db_conn_rw: &Connection,
275288
write_block: bool,
276289
hord_db_path: &PathBuf,
290+
traversals_cache: &Arc<
291+
DashMap<
292+
(u32, [u8; 8]),
293+
(Vec<([u8; 8], u32, u16, u64)>, Vec<u64>),
294+
BuildHasherDefault<FxHasher>,
295+
>,
296+
>,
277297
ctx: &Context,
278298
) -> Result<(), String> {
279299
if write_block {
@@ -299,6 +319,7 @@ pub fn update_hord_db_and_augment_bitcoin_block(
299319
&new_block,
300320
Some(inscriptions_db_conn_rw),
301321
hord_db_path,
322+
traversals_cache,
302323
ctx,
303324
);
304325

‎components/chainhook-event-observer/src/observer/mod.rs

+34-25
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use crate::chainhooks::types::{
1010
ChainhookConfig, ChainhookFullSpecification, ChainhookSpecification,
1111
};
1212

13+
use crate::hord::new_traversals_cache;
1314
#[cfg(feature = "ordinals")]
1415
use crate::hord::{
1516
db::{open_readwrite_hord_db_conn, open_readwrite_hord_db_conn_rocks_db},
@@ -695,21 +696,25 @@ pub async fn start_observer_commands_handler(
695696
match bitcoin_block_store.get_mut(&header.block_identifier) {
696697
Some(block) => {
697698
#[cfg(feature = "ordinals")]
698-
if let Err(e) = update_hord_db_and_augment_bitcoin_block(
699-
block,
700-
&blocks_db,
701-
&inscriptions_db_conn_rw,
702-
true,
703-
&config.get_cache_path_buf(),
704-
&ctx,
705-
) {
706-
ctx.try_log(|logger| {
707-
slog::error!(
699+
{
700+
let traversals_cache = Arc::new(new_traversals_cache());
701+
if let Err(e) = update_hord_db_and_augment_bitcoin_block(
702+
block,
703+
&blocks_db,
704+
&inscriptions_db_conn_rw,
705+
true,
706+
&config.get_cache_path_buf(),
707+
&traversals_cache,
708+
&ctx,
709+
) {
710+
ctx.try_log(|logger| {
711+
slog::error!(
708712
logger,
709713
"Unable to insert bitcoin block {} in hord_db: {e}",
710714
block.block_identifier.index
711715
)
712-
});
716+
});
717+
}
713718
}
714719
new_blocks.push(block.clone());
715720
}
@@ -833,20 +838,24 @@ pub async fn start_observer_commands_handler(
833838
match bitcoin_block_store.get_mut(&header.block_identifier) {
834839
Some(block) => {
835840
#[cfg(feature = "ordinals")]
836-
if let Err(e) = update_hord_db_and_augment_bitcoin_block(
837-
block,
838-
&blocks_db,
839-
&inscriptions_db_conn_rw,
840-
true,
841-
&config.get_cache_path_buf(),
842-
&ctx,
843-
) {
844-
ctx.try_log(|logger| {
845-
slog::error!(
846-
logger,
847-
"Unable to apply bitcoin block {} with hord_db: {e}", block.block_identifier.index
848-
)
849-
});
841+
{
842+
let traversals_cache = Arc::new(new_traversals_cache());
843+
if let Err(e) = update_hord_db_and_augment_bitcoin_block(
844+
block,
845+
&blocks_db,
846+
&inscriptions_db_conn_rw,
847+
true,
848+
&config.get_cache_path_buf(),
849+
&traversals_cache,
850+
&ctx,
851+
) {
852+
ctx.try_log(|logger| {
853+
slog::error!(
854+
logger,
855+
"Unable to apply bitcoin block {} with hord_db: {e}", block.block_identifier.index
856+
)
857+
});
858+
}
850859
}
851860
blocks_to_apply.push(block.clone());
852861
}

0 commit comments

Comments
 (0)