feat: polish first impression

Ludo Galabru · Ludo Galabru · commit 3c2b00ce38f8 · 2023-07-20T17:31:52.000-04:00
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/components/hord-cli/Cargo.toml b/components/hord-cli/Cargo.toml
@@ -39,6 +39,7 @@ rusqlite = { version = "0.27.0", features = ["bundled"] }
 anyhow = { version = "1.0.56", features = ["backtrace"] }
 schemars = { version = "0.8.10", git = "https://github.com/hirosystems/schemars.git", branch = "feat-chainhook-fixes" }
 pprof = { version = "0.12", features = ["flamegraph"], optional = true }
+progressing = '3'
 
 [dependencies.rocksdb]
 version = "0.20.1"
diff --git a/components/hord-cli/src/archive/mod.rs b/components/hord-cli/src/archive/mod.rs
@@ -1,12 +1,14 @@
 use crate::config::Config;
-use crate::utils::{read_file_content_at_path, write_file_content_at_path};
+use crate::utils::read_file_content_at_path;
 use chainhook_sdk::types::BitcoinNetwork;
 use chainhook_sdk::utils::Context;
 use flate2::read::GzDecoder;
 use futures_util::StreamExt;
 use std::fs;
 use std::io::{self, Cursor};
 use std::io::{Read, Write};
+use progressing::Baring;
+use progressing::mapping::Bar as MappingBar;
 
 pub fn default_sqlite_file_path(_network: &BitcoinNetwork) -> String {
     format!("hord.sqlite").to_lowercase()
@@ -16,28 +18,28 @@ pub fn default_sqlite_sha_file_path(_network: &BitcoinNetwork) -> String {
     format!("hord.sqlite.sha256").to_lowercase()
 }
 
-pub async fn download_sqlite_file(config: &Config) -> Result<(), String> {
+pub async fn download_sqlite_file(config: &Config, ctx: &Context) -> Result<(), String> {
     let mut destination_path = config.expected_cache_path();
     std::fs::create_dir_all(&destination_path).unwrap_or_else(|e| {
         println!("{}", e.to_string());
     });
 
-    let remote_sha_url = config.expected_remote_ordinals_sqlite_sha256();
-    let res = reqwest::get(&remote_sha_url)
-        .await
-        .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?
-        .bytes()
-        .await
-        .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?;
-
-    let mut local_sha_file_path = destination_path.clone();
-    local_sha_file_path.push(default_sqlite_sha_file_path(
-        &config.network.bitcoin_network,
-    ));
+    // let remote_sha_url = config.expected_remote_ordinals_sqlite_sha256();
+    // let res = reqwest::get(&remote_sha_url)
+    //     .await
+    //     .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?
+    //     .bytes()
+    //     .await
+    //     .or(Err(format!("Failed to GET from '{}'", &remote_sha_url)))?;
 
-    write_file_content_at_path(&local_sha_file_path, &res.to_vec())?;
+    // let mut local_sha_file_path = destination_path.clone();
+    // local_sha_file_path.push(default_sqlite_sha_file_path(
+    //     &config.network.bitcoin_network,
+    // ));
+    // write_file_content_at_path(&local_sha_file_path, &res.to_vec())?;
 
     let file_url = config.expected_remote_ordinals_sqlite_url();
+    println!("=> {file_url}");
     let res = reqwest::get(&file_url)
         .await
         .or(Err(format!("Failed to GET from '{}'", &file_url)))?;
@@ -59,9 +61,25 @@ pub async fn download_sqlite_file(config: &Config) -> Result<(), String> {
     });
 
     if res.status() == reqwest::StatusCode::OK {
+        let mut progress_bar = MappingBar::with_range(0i64, 4_800_001_704);
+        progress_bar.set_len(40);
+        info!(
+            ctx.expect_logger(),
+            "{}", progress_bar
+        );
         let mut stream = res.bytes_stream();
+        let mut progress = 0;
         while let Some(item) = stream.next().await {
             let chunk = item.or(Err(format!("Error while downloading file")))?;
+            progress += chunk.len() as i64;
+            progress_bar.set(progress);
+            if progress_bar.has_progressed_significantly() {
+                progress_bar.remember_significant_progress();
+                info!(
+                    ctx.expect_logger(),
+                    "{}", progress_bar
+                );
+            }
             tx.send_async(chunk.to_vec())
                 .await
                 .map_err(|e| format!("unable to download stacks event: {}", e.to_string()))?;
@@ -116,14 +134,14 @@ pub async fn download_ordinals_dataset_if_required(config: &Config, ctx: &Contex
             let url = config.expected_remote_ordinals_sqlite_url();
             let mut sqlite_file_path = config.expected_cache_path();
             sqlite_file_path.push(default_sqlite_file_path(&config.network.bitcoin_network));
-            let mut tsv_sha_file_path = config.expected_cache_path();
-            tsv_sha_file_path.push(default_sqlite_sha_file_path(
+            let mut sqlite_sha_file_path = config.expected_cache_path();
+            sqlite_sha_file_path.push(default_sqlite_sha_file_path(
                 &config.network.bitcoin_network,
             ));
 
             // Download archive if not already present in cache
             // Load the local
-            let local_sha_file = read_file_content_at_path(&tsv_sha_file_path);
+            let local_sha_file = read_file_content_at_path(&sqlite_sha_file_path);
             let sha_url = config.expected_remote_ordinals_sqlite_sha256();
 
             let remote_sha_file = match reqwest::get(&sha_url).await {
@@ -136,22 +154,27 @@ pub async fn download_ordinals_dataset_if_required(config: &Config, ctx: &Contex
                     if cache_not_expired {
                         info!(
                             ctx.expect_logger(),
-                            "More recent Stacks archive file detected"
+                            "More recent hord.sqlite file detected"
                         );
                     }
                     cache_not_expired == false
                 }
                 (_, _) => {
-                    info!(
-                        ctx.expect_logger(),
-                        "Unable to retrieve Stacks archive file locally"
-                    );
-                    true
+                    match std::fs::metadata(&sqlite_file_path) {
+                        Ok(_) => false,
+                        _ => {
+                            info!(
+                                ctx.expect_logger(),
+                                "Unable to retrieve hord.sqlite file locally"
+                            );
+                            true        
+                        }
+                    }
                 }
             };
             if should_download {
                 info!(ctx.expect_logger(), "Downloading {}", url);
-                match download_sqlite_file(&config).await {
+                match download_sqlite_file(&config, &ctx).await {
                     Ok(_) => {}
                     Err(e) => {
                         error!(ctx.expect_logger(), "{}", e);
diff --git a/components/hord-cli/src/cli/mod.rs b/components/hord-cli/src/cli/mod.rs
@@ -1,3 +1,4 @@
+use crate::archive::download_ordinals_dataset_if_required;
 use crate::config::generator::generate_config;
 use crate::config::Config;
 use crate::scan::bitcoin::scan_bitcoin_chainstate_via_rpc_using_predicate;
@@ -455,9 +456,13 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
                     .get_sorted_entries();
 
             if let Some(ref post_to) = cmd.post_to {
+                info!(ctx.expect_logger(), "A fully synchronized bitcoind node is required for retrieving inscriptions content.");
+                info!(ctx.expect_logger(), "Checking {}...", config.network.bitcoind_rpc_url);
                 let tip = check_bitcoind_connection(&config).await?;
                 if tip < cmd.end_block {
-                    error!(ctx.expect_logger(), "unable to scan block range [{}, {}]: underlying bitcoind synchronized until block {} ", cmd.start_block, cmd.end_block, tip);
+                    error!(ctx.expect_logger(), "Unable to scan block range [{}, {}]: underlying bitcoind synchronized until block {} ", cmd.start_block, cmd.end_block, tip);
+                } else {
+                    info!(ctx.expect_logger(), "Starting scan");
                 }
 
                 let predicate_spec = build_predicate_from_cli(
@@ -470,6 +475,8 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
                 scan_bitcoin_chainstate_via_rpc_using_predicate(&predicate_spec, &config, &ctx)
                     .await?;
             } else {
+                let _ = download_ordinals_dataset_if_required(&config, ctx).await;
+
                 let inscriptions_db_conn =
                     open_readonly_hord_db_conn(&config.expected_cache_path(), &ctx)?;
                 while let Some(block_height) = block_range.pop_front() {
@@ -501,12 +508,13 @@ async fn handle_command(opts: Opts, ctx: &Context) -> Result<(), String> {
                             );
                         }
                     }
-                    println!(
-                        "Inscriptions revealed: {}, inscriptions transferred: {total_transfers}",
-                        inscriptions.len()
-                    );
-
-                    println!("-----");
+                    if total_transfers > 0 && inscriptions.len() > 0 {
+                        println!(
+                            "Inscriptions revealed: {}, inscriptions transferred: {total_transfers}",
+                            inscriptions.len()
+                        );
+                        println!("-----");    
+                    }
                 }
             }
         }
diff --git a/components/hord-cli/src/config/mod.rs b/components/hord-cli/src/config/mod.rs
@@ -12,12 +12,8 @@ use std::fs::File;
 use std::io::{BufReader, Read};
 use std::path::PathBuf;
 
-const DEFAULT_MAINNET_STACKS_TSV_ARCHIVE: &str =
-    "https://archive.hiro.so/mainnet/stacks-blockchain-api/mainnet-stacks-blockchain-api-latest";
-const DEFAULT_TESTNET_STACKS_TSV_ARCHIVE: &str =
-    "https://archive.hiro.so/testnet/stacks-blockchain-api/testnet-stacks-blockchain-api-latest";
 const DEFAULT_MAINNET_ORDINALS_SQLITE_ARCHIVE: &str =
-    "https://archive.hiro.so/mainnet/chainhooks/hord-latest.sqlite";
+    "https://archive.hiro.so/mainnet/chainhooks/hord.sqlite";
 const DEFAULT_REDIS_URI: &str = "redis://localhost:6379/";
 
 pub const DEFAULT_INGESTION_PORT: u16 = 20455;
@@ -56,8 +52,6 @@ pub struct PredicatesApiConfig {
 
 #[derive(Clone, Debug)]
 pub enum EventSourceConfig {
-    StacksTsvPath(PathConfig),
-    StacksTsvUrl(UrlConfig),
     OrdinalsSqlitePath(PathConfig),
     OrdinalsSqliteUrl(UrlConfig),
 }
@@ -154,11 +148,11 @@ impl Config {
             if let Some(dst) = source.tsv_file_path.take() {
                 let mut file_path = PathBuf::new();
                 file_path.push(dst);
-                event_sources.push(EventSourceConfig::StacksTsvPath(PathConfig { file_path }));
+                event_sources.push(EventSourceConfig::OrdinalsSqlitePath(PathConfig { file_path }));
                 continue;
             }
             if let Some(file_url) = source.tsv_file_url.take() {
-                event_sources.push(EventSourceConfig::StacksTsvUrl(UrlConfig { file_url }));
+                event_sources.push(EventSourceConfig::OrdinalsSqliteUrl(UrlConfig { file_url }));
                 continue;
             }
         }
@@ -234,22 +228,14 @@ impl Config {
     pub fn is_initial_ingestion_required(&self) -> bool {
         for source in self.event_sources.iter() {
             match source {
-                EventSourceConfig::StacksTsvUrl(_) | EventSourceConfig::StacksTsvPath(_) => {
+                EventSourceConfig::OrdinalsSqlitePath(_) | EventSourceConfig::OrdinalsSqliteUrl(_) => {
                     return true
                 }
-                _ => {}
             }
         }
         return false;
     }
 
-    pub fn add_local_stacks_tsv_source(&mut self, file_path: &PathBuf) {
-        self.event_sources
-            .push(EventSourceConfig::StacksTsvPath(PathConfig {
-                file_path: file_path.clone(),
-            }));
-    }
-
     pub fn add_ordinals_sqlite_remote_source_url(&mut self, file_url: &str) {
         self.event_sources
             .push(EventSourceConfig::OrdinalsSqliteUrl(UrlConfig {
@@ -275,15 +261,6 @@ impl Config {
         }
     }
 
-    pub fn expected_local_stacks_tsv_file(&self) -> &PathBuf {
-        for source in self.event_sources.iter() {
-            if let EventSourceConfig::StacksTsvPath(config) = source {
-                return &config.file_path;
-            }
-        }
-        panic!("expected local-tsv source")
-    }
-
     pub fn expected_cache_path(&self) -> PathBuf {
         let mut destination_path = PathBuf::new();
         destination_path.push(&self.storage.working_dir);
@@ -299,23 +276,6 @@ impl Config {
         panic!("expected remote-tsv source")
     }
 
-    fn expected_remote_stacks_tsv_base_url(&self) -> &String {
-        for source in self.event_sources.iter() {
-            if let EventSourceConfig::StacksTsvUrl(config) = source {
-                return &config.file_url;
-            }
-        }
-        panic!("expected remote-tsv source")
-    }
-
-    pub fn expected_remote_stacks_tsv_sha256(&self) -> String {
-        format!("{}.sha256", self.expected_remote_stacks_tsv_base_url())
-    }
-
-    pub fn expected_remote_stacks_tsv_url(&self) -> String {
-        format!("{}.gz", self.expected_remote_stacks_tsv_base_url())
-    }
-
     pub fn expected_remote_ordinals_sqlite_sha256(&self) -> String {
         format!("{}.sha256", self.expected_remote_ordinals_sqlite_base_url())
     }
@@ -324,15 +284,6 @@ impl Config {
         format!("{}.gz", self.expected_remote_ordinals_sqlite_base_url())
     }
 
-    pub fn rely_on_remote_stacks_tsv(&self) -> bool {
-        for source in self.event_sources.iter() {
-            if let EventSourceConfig::StacksTsvUrl(_config) = source {
-                return true;
-            }
-        }
-        false
-    }
-
     pub fn rely_on_remote_ordinals_sqlite(&self) -> bool {
         for source in self.event_sources.iter() {
             if let EventSourceConfig::OrdinalsSqliteUrl(_config) = source {
@@ -342,20 +293,6 @@ impl Config {
         false
     }
 
-    pub fn should_download_remote_stacks_tsv(&self) -> bool {
-        let mut rely_on_remote_tsv = false;
-        let mut remote_tsv_present_locally = false;
-        for source in self.event_sources.iter() {
-            if let EventSourceConfig::StacksTsvUrl(_config) = source {
-                rely_on_remote_tsv = true;
-            }
-            if let EventSourceConfig::StacksTsvPath(_config) = source {
-                remote_tsv_present_locally = true;
-            }
-        }
-        rely_on_remote_tsv == true && remote_tsv_present_locally == false
-    }
-
     pub fn should_download_remote_ordinals_sqlite(&self) -> bool {
         let mut rely_on_remote_tsv = false;
         let mut remote_tsv_present_locally = false;
@@ -421,9 +358,7 @@ impl Config {
                 working_dir: default_cache_path(),
             },
             http_api: PredicatesApi::Off,
-            event_sources: vec![EventSourceConfig::StacksTsvUrl(UrlConfig {
-                file_url: DEFAULT_TESTNET_STACKS_TSV_ARCHIVE.into(),
-            })],
+            event_sources: vec![],
             limits: LimitsConfig {
                 max_number_of_bitcoin_predicates: BITCOIN_MAX_PREDICATE_REGISTRATION,
                 max_number_of_concurrent_bitcoin_scans: BITCOIN_SCAN_THREAD_POOL_SIZE,
@@ -453,9 +388,6 @@ impl Config {
             },
             http_api: PredicatesApi::Off,
             event_sources: vec![
-                EventSourceConfig::StacksTsvUrl(UrlConfig {
-                    file_url: DEFAULT_MAINNET_STACKS_TSV_ARCHIVE.into(),
-                }),
                 EventSourceConfig::OrdinalsSqliteUrl(UrlConfig {
                     file_url: DEFAULT_MAINNET_ORDINALS_SQLITE_ARCHIVE.into(),
                 }),
diff --git a/components/hord-cli/src/db/mod.rs b/components/hord-cli/src/db/mod.rs
@@ -1,6 +1,5 @@
 use std::{
     collections::{BTreeMap, HashMap},
-    fs::File,
     hash::BuildHasherDefault,
     path::PathBuf,
     sync::{mpsc::Sender, Arc},
