feat: ability for chainhook-node to start with custom config

components/chainhook-node/Chainhook.toml

@@ -0,0 +1,14 @@
+[storage]
+driver = "redis"
+redis_uri = "redis://localhost:6379/"
+
+[chainhooks]
+max_stacks_registrations = 500
+max_bitcoin_registrations = 500
+
+[network]
+mode = "devnet"
+bitcoin_node_rpc_url = "http://0.0.0.0:18443"
+bitcoin_node_rpc_username = "devnet"
+bitcoin_node_rpc_password = "devnet"
+stacks_node_rpc_url = "http://0.0.0.0:20443"

components/chainhook-node/src/block/digestion.rs

@@ -2,7 +2,6 @@ use super::DigestingCommand;
 use crate::config::Config;
 use chainhook_event_observer::indexer;
 use chainhook_event_observer::indexer::Indexer;
-use redis;
 use redis::Commands;
 use std::cmp::Ordering;
 use std::{collections::BinaryHeap, process, sync::mpsc::Receiver};
@@ -23,7 +22,7 @@ pub fn start(command_rx: Receiver<DigestingCommand>, config: &Config) -> Result<
     let mut job_queue: BinaryHeap<Job> = BinaryHeap::new();
     let redis_config = config.expected_redis_config();
     let client = redis::Client::open(redis_config.uri.clone()).unwrap();
-    let mut indexer = Indexer::new(config.indexer.clone());
+    let mut indexer = Indexer::new(config.network.clone());
 
     let mut con = match client.get_connection() {
         Ok(con) => con,

components/chainhook-node/src/block/ingestion.rs

@@ -1,7 +1,6 @@
 use crate::config::Config;
 use chainhook_event_observer::indexer::{self, Indexer};
 use chainhook_types::BlockIdentifier;
-use redis;
 use redis::Commands;
 use serde::Deserialize;
 use std::sync::mpsc::Sender;
@@ -94,7 +93,7 @@ pub fn start(
                 return Err(format!("Redis: {}", message.to_string()));
             }
         };
-        let _indexer = Indexer::new(stacks_thread_config.indexer.clone());
+        let _indexer = Indexer::new(stacks_thread_config.network.clone());
 
         // Retrieve the former highest block height stored
         let former_tip_height: u64 = con.get(&format!("stx:tip")).unwrap_or(0);

components/chainhook-node/src/cli/mod.rs

@@ -3,6 +3,7 @@ use crate::archive;
 use crate::block::DigestingCommand;
 use crate::config::Config;
 
+use chainhook_db::config::ConfigFile;
 use chainhook_event_observer::chainhooks::bitcoin::{
     handle_bitcoin_hook_action, BitcoinChainhookOccurrence, BitcoinTriggerChainhook,
 };
@@ -74,6 +75,14 @@ struct StartNode {
         conflicts_with = "devnet"
     )]
     pub mainnet: bool,
+    /// Load config file path
+    #[clap(
+        long = "config-path",
+        conflicts_with = "mainnet",
+        conflicts_with = "testnet",
+        conflicts_with = "devnet"
+    )]
+    pub config_path: Option<String>,
 }
 
 #[derive(Parser, PartialEq, Clone, Debug)]
@@ -117,19 +126,26 @@ pub fn main() {
 
     match opts.command {
         Command::Start(cmd) => {
-            let network = match (cmd.devnet, cmd.testnet, cmd.mainnet) {
-                (true, false, false) => StacksNetwork::Devnet,
-                (false, true, false) => StacksNetwork::Testnet,
-                (false, false, true) => StacksNetwork::Mainnet,
+            let config = match (cmd.devnet, cmd.testnet, cmd.mainnet, cmd.config_path) {
+                (true, false, false, _) => Config::devnet_default(),
+                (false, true, false, _) => Config::testnet_default(),
+                (false, false, true, _) => Config::mainnet_default(),
+                (false, false, false, Some(config_path)) => {
+                    match Config::from_file_path(&config_path) {
+                        Ok(config) => config,
+                        Err(e) => {
+                            println!("{e}");
+                            process::exit(1);
+                        }
+                    }
+                }
                 _ => {
-                    println!(
-                        "{}",
-                        format_err!("network flag required (devnet, testnet, mainnet)")
-                    );
+                    println!("network flag required (devnet, testnet, mainnet)");
                     process::exit(1);
                 }
             };
-            start_node(&network);
+            println!("{:?}", config);
+            start_node(config);
         }
         Command::Replay(cmd) => {
             let network = match (cmd.devnet, cmd.testnet, cmd.mainnet) {
@@ -191,9 +207,9 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply:
     let bitcoin_port = bitcoind_rpc_url
         .port()
         .expect("unable to retrieve port from bitcoin_url");
-    config.indexer.bitcoin_node_rpc_url = format!("http://{}:{}", bitcoin_host, bitcoin_port);
-    config.indexer.bitcoin_node_rpc_username = bitcoind_rpc_url.username().to_string();
-    config.indexer.bitcoin_node_rpc_password = bitcoind_rpc_url
+    config.network.bitcoin_node_rpc_url = format!("http://{}:{}", bitcoin_host, bitcoin_port);
+    config.network.bitcoin_node_rpc_username = bitcoind_rpc_url.username().to_string();
+    config.network.bitcoin_node_rpc_password = bitcoind_rpc_url
         .password()
         .expect("unable to retrieve password from bitcoin_url")
         .to_string();
@@ -258,9 +274,9 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply:
         initial_hook_formation: None,
         ingestion_port: DEFAULT_INGESTION_PORT,
         control_port: DEFAULT_CONTROL_PORT,
-        bitcoin_node_username: config.indexer.bitcoin_node_rpc_username.clone(),
-        bitcoin_node_password: config.indexer.bitcoin_node_rpc_password.clone(),
-        bitcoin_node_rpc_host: config.indexer.bitcoin_node_rpc_url.clone(),
+        bitcoin_node_username: config.network.bitcoin_node_rpc_username.clone(),
+        bitcoin_node_password: config.network.bitcoin_node_rpc_password.clone(),
+        bitcoin_node_rpc_host: config.network.bitcoin_node_rpc_url.clone(),
         bitcoin_node_rpc_port: bitcoin_port,
         stacks_node_rpc_host: "http://localhost".into(),
         stacks_node_rpc_port: 20443,
@@ -296,10 +312,10 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply:
     };
 
     let auth = Auth::UserPass(
-        config.indexer.bitcoin_node_rpc_username.clone(),
-        config.indexer.bitcoin_node_rpc_password.clone(),
+        config.network.bitcoin_node_rpc_username.clone(),
+        config.network.bitcoin_node_rpc_password.clone(),
     );
-    let bitcoin_rpc = Client::new(&config.indexer.bitcoin_node_rpc_url, auth).unwrap();
+    let bitcoin_rpc = Client::new(&config.network.bitcoin_node_rpc_url, auth).unwrap();
 
     loop {
         let event = match observer_event_rx.recv() {
@@ -504,7 +520,7 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply:
                                     };
 
                                     let block = match bitcoin_rpc.get_block(&block_hash) {
-                                        Ok(block) => build_block(block, cursor, &config.indexer),
+                                        Ok(block) => build_block(block, cursor, &config.network),
                                         Err(e) => {
                                             error!("unable to retrieve block hash {}", cursor);
                                             continue;
@@ -611,7 +627,7 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply:
     }
 }
 
-pub fn start_node(network: &StacksNetwork) {
+pub fn start_node(mut config: Config) {
     let (digestion_tx, digestion_rx) = channel();
     let (observer_event_tx, observer_event_rx) = channel();
     let (observer_command_tx, observer_command_rx) = channel();
@@ -625,13 +641,6 @@ pub fn start_node(network: &StacksNetwork) {
     })
     .expect("Error setting Ctrl-C handler");
 
-    let mut config = match network {
-        StacksNetwork::Devnet => Config::devnet_default(),
-        StacksNetwork::Testnet => Config::testnet_default(),
-        StacksNetwork::Mainnet => Config::mainnet_default(),
-        _ => unreachable!(),
-    };
-
     if config.is_initial_ingestion_required() {
         // Download default tsv.
         if config.rely_on_remote_tsv() && config.should_download_remote_tsv() {
@@ -692,8 +701,8 @@ pub fn start_node(network: &StacksNetwork) {
         initial_hook_formation: None,
         ingestion_port: DEFAULT_INGESTION_PORT,
         control_port: DEFAULT_CONTROL_PORT,
-        bitcoin_node_username: config.indexer.bitcoin_node_rpc_username.clone(),
-        bitcoin_node_password: config.indexer.bitcoin_node_rpc_password.clone(),
+        bitcoin_node_username: config.network.bitcoin_node_rpc_username.clone(),
+        bitcoin_node_password: config.network.bitcoin_node_rpc_password.clone(),
         bitcoin_node_rpc_host: "http://localhost".into(),
         bitcoin_node_rpc_port: 18443,
         stacks_node_rpc_host: "http://localhost".into(),

components/chainhook-node/src/config/file.rs

@@ -1,17 +1,18 @@
-#[derive(Clone, Debug)]
+#[derive(Deserialize, Debug, Clone)]
 pub struct ConfigFile {
-    pub storage: Option<StorageConfigFile>,
-    pub event_source: Vec<EventSourceConfigFile>,
-    pub chainhooks: Option<ChainhooksConfigFile>,
+    pub storage: StorageConfigFile,
+    pub event_source: Option<Vec<EventSourceConfigFile>>,
+    pub chainhooks: ChainhooksConfigFile,
+    pub network: NetworkConfigFile,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Deserialize, Debug, Clone)]
 pub struct StorageConfigFile {
-    pub driver: Option<String>,
-    pub redis_uri: Option<String>,
+    pub driver: String,
+    pub redis_uri: String,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Deserialize, Debug, Clone)]
 pub struct EventSourceConfigFile {
     pub source_type: Option<String>,
     pub stacks_node_url: Option<String>,
@@ -21,8 +22,17 @@ pub struct EventSourceConfigFile {
     pub tsv_file_url: Option<String>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Deserialize, Debug, Clone)]
 pub struct ChainhooksConfigFile {
     pub max_stacks_registrations: Option<u16>,
     pub max_bitcoin_registrations: Option<u16>,
 }
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct NetworkConfigFile {
+    pub mode: String,
+    pub bitcoin_node_rpc_url: String,
+    pub bitcoin_node_rpc_username: String,
+    pub bitcoin_node_rpc_password: String,
+    pub stacks_node_rpc_url: String,
+}

components/chainhook-node/src/config/mod.rs

@@ -3,17 +3,19 @@ pub mod file;
 pub use chainhook_event_observer::indexer::IndexerConfig;
 use chainhook_types::{BitcoinNetwork, StacksNetwork};
 pub use file::ConfigFile;
+use std::fs::File;
+use std::io::{BufReader, Read};
 use std::path::PathBuf;
 
-const DEFAULT_MAINNET_TSV_ARCHIVE: &str = "https://storage.googleapis.com/hirosystems-archive/mainnet/api/mainnet-blockchain-api-5.0.3-latest.tar.gz";
-const DEFAULT_TESTNET_TSV_ARCHIVE: &str = "https://storage.googleapis.com/hirosystems-archive/testnet/api/testnet-blockchain-api-5.0.3-latest.tar.gz";
+const DEFAULT_MAINNET_TSV_ARCHIVE: &str = "https://storage.googleapis.com/hirosystems-archive/mainnet/api/mainnet-blockchain-api-latest.tar.gz";
+const DEFAULT_TESTNET_TSV_ARCHIVE: &str = "https://storage.googleapis.com/hirosystems-archive/testnet/api/testnet-blockchain-api-latest.tar.gz";
 
 #[derive(Clone, Debug)]
 pub struct Config {
     pub storage: StorageConfig,
     pub event_sources: Vec<EventSourceConfig>,
     pub chainhooks: ChainhooksConfig,
-    pub indexer: IndexerConfig,
+    pub network: IndexerConfig,
 }
 
 #[derive(Clone, Debug)]
@@ -61,6 +63,70 @@ pub struct ChainhooksConfig {
 }
 
 impl Config {
+    pub fn from_file_path(file_path: &str) -> Result<Config, String> {
+        let file = File::open(file_path)
+            .map_err(|e| format!("unable to read file {}\n{:?}", file_path, e))?;
+        let mut file_reader = BufReader::new(file);
+        let mut file_buffer = vec![];
+        file_reader
+            .read_to_end(&mut file_buffer)
+            .map_err(|e| format!("unable to read file {}\n{:?}", file_path, e))?;
+
+        let config_file: ConfigFile = match toml::from_slice(&file_buffer) {
+            Ok(s) => s,
+            Err(e) => {
+                return Err(format!("Config file malformatted {}", e.to_string()));
+            }
+        };
+        Config::from_config_file(config_file)
+    }
+
+    pub fn from_config_file(config_file: ConfigFile) -> Result<Config, String> {
+        let (stacks_network, bitcoin_network) = match config_file.network.mode.as_str() {
+            "devnet" => (StacksNetwork::Devnet, BitcoinNetwork::Regtest),
+            "testnet" => (StacksNetwork::Testnet, BitcoinNetwork::Testnet),
+            "mainnet" => (StacksNetwork::Mainnet, BitcoinNetwork::Mainnet),
+            _ => return Err("network.mode not supported".to_string()),
+        };
+
+        let config = Config {
+            storage: StorageConfig {
+                driver: StorageDriver::Redis(RedisConfig {
+                    uri: config_file.storage.redis_uri.to_string(),
+                }),
+                cache_path: "cache".into(),
+            },
+            event_sources: vec![EventSourceConfig::StacksNode(StacksNodeConfig {
+                host: config_file.network.stacks_node_rpc_url.to_string(),
+            })],
+            chainhooks: ChainhooksConfig {
+                max_stacks_registrations: config_file
+                    .chainhooks
+                    .max_stacks_registrations
+                    .unwrap_or(100),
+                max_bitcoin_registrations: config_file
+                    .chainhooks
+                    .max_bitcoin_registrations
+                    .unwrap_or(100),
+            },
+            network: IndexerConfig {
+                stacks_node_rpc_url: config_file.network.stacks_node_rpc_url.to_string(),
+                bitcoin_node_rpc_url: config_file.network.bitcoin_node_rpc_url.to_string(),
+                bitcoin_node_rpc_username: config_file
+                    .network
+                    .bitcoin_node_rpc_username
+                    .to_string(),
+                bitcoin_node_rpc_password: config_file
+                    .network
+                    .bitcoin_node_rpc_password
+                    .to_string(),
+                stacks_network,
+                bitcoin_network,
+            },
+        };
+        Ok(config)
+    }
+
     pub fn is_initial_ingestion_required(&self) -> bool {
         for source in self.event_sources.iter() {
             match source {
@@ -155,7 +221,7 @@ impl Config {
                 max_stacks_registrations: 50,
                 max_bitcoin_registrations: 50,
             },
-            indexer: IndexerConfig {
+            network: IndexerConfig {
                 stacks_node_rpc_url: "http://0.0.0.0:20443".into(),
                 bitcoin_node_rpc_url: "http://0.0.0.0:18443".into(),
                 bitcoin_node_rpc_username: "devnet".into(),
@@ -181,7 +247,7 @@ impl Config {
                 max_stacks_registrations: 10,
                 max_bitcoin_registrations: 10,
             },
-            indexer: IndexerConfig {
+            network: IndexerConfig {
                 stacks_node_rpc_url: "http://0.0.0.0:20443".into(),
                 bitcoin_node_rpc_url: "http://0.0.0.0:18443".into(),
                 bitcoin_node_rpc_username: "devnet".into(),
@@ -207,7 +273,7 @@ impl Config {
                 max_stacks_registrations: 10,
                 max_bitcoin_registrations: 10,
             },
-            indexer: IndexerConfig {
+            network: IndexerConfig {
                 stacks_node_rpc_url: "http://0.0.0.0:20443".into(),
                 bitcoin_node_rpc_url: "http://0.0.0.0:18443".into(),
                 bitcoin_node_rpc_username: "devnet".into(),

components/chainhook-node/src/lib.rs

@@ -4,8 +4,10 @@ extern crate serde_json;
 #[macro_use]
 extern crate slog_scope;
 
-extern crate serde;
+#[macro_use]
 extern crate serde_derive;
 
+extern crate serde;
+
 pub mod block;
 pub mod config;

components/chainhook-node/src/main.rs

@@ -7,8 +7,10 @@ extern crate slog_scope;
 #[macro_use]
 extern crate hiro_system_kit;
 
-extern crate serde;
+#[macro_use]
 extern crate serde_derive;
+
+extern crate serde;
 extern crate slog;
 
 use slog_async;