diff --git a/components/chainhook-node/Chainhook.toml b/components/chainhook-node/Chainhook.toml new file mode 100644 index 000000000..3d4db4b07 --- /dev/null +++ b/components/chainhook-node/Chainhook.toml @@ -0,0 +1,14 @@ +[storage] +driver = "redis" +redis_uri = "redis://localhost:6379/" + +[chainhooks] +max_stacks_registrations = 500 +max_bitcoin_registrations = 500 + +[network] +mode = "devnet" +bitcoin_node_rpc_url = "http://0.0.0.0:18443" +bitcoin_node_rpc_username = "devnet" +bitcoin_node_rpc_password = "devnet" +stacks_node_rpc_url = "http://0.0.0.0:20443" diff --git a/components/chainhook-node/src/block/digestion.rs b/components/chainhook-node/src/block/digestion.rs index 08198a737..4a67b8005 100644 --- a/components/chainhook-node/src/block/digestion.rs +++ b/components/chainhook-node/src/block/digestion.rs @@ -2,7 +2,6 @@ use super::DigestingCommand; use crate::config::Config; use chainhook_event_observer::indexer; use chainhook_event_observer::indexer::Indexer; -use redis; use redis::Commands; use std::cmp::Ordering; use std::{collections::BinaryHeap, process, sync::mpsc::Receiver}; @@ -23,7 +22,7 @@ pub fn start(command_rx: Receiver, config: &Config) -> Result< let mut job_queue: BinaryHeap = BinaryHeap::new(); let redis_config = config.expected_redis_config(); let client = redis::Client::open(redis_config.uri.clone()).unwrap(); - let mut indexer = Indexer::new(config.indexer.clone()); + let mut indexer = Indexer::new(config.network.clone()); let mut con = match client.get_connection() { Ok(con) => con, diff --git a/components/chainhook-node/src/block/ingestion.rs b/components/chainhook-node/src/block/ingestion.rs index f07245685..a6c99d150 100644 --- a/components/chainhook-node/src/block/ingestion.rs +++ b/components/chainhook-node/src/block/ingestion.rs @@ -1,7 +1,6 @@ use crate::config::Config; use chainhook_event_observer::indexer::{self, Indexer}; use chainhook_types::BlockIdentifier; -use redis; use redis::Commands; use serde::Deserialize; use std::sync::mpsc::Sender; @@ -94,7 +93,7 @@ pub fn start( return Err(format!("Redis: {}", message.to_string())); } }; - let _indexer = Indexer::new(stacks_thread_config.indexer.clone()); + let _indexer = Indexer::new(stacks_thread_config.network.clone()); // Retrieve the former highest block height stored let former_tip_height: u64 = con.get(&format!("stx:tip")).unwrap_or(0); diff --git a/components/chainhook-node/src/cli/mod.rs b/components/chainhook-node/src/cli/mod.rs index 1f65fc1e1..75253a2fc 100644 --- a/components/chainhook-node/src/cli/mod.rs +++ b/components/chainhook-node/src/cli/mod.rs @@ -3,6 +3,7 @@ use crate::archive; use crate::block::DigestingCommand; use crate::config::Config; +use chainhook_db::config::ConfigFile; use chainhook_event_observer::chainhooks::bitcoin::{ handle_bitcoin_hook_action, BitcoinChainhookOccurrence, BitcoinTriggerChainhook, }; @@ -74,6 +75,14 @@ struct StartNode { conflicts_with = "devnet" )] pub mainnet: bool, + /// Load config file path + #[clap( + long = "config-path", + conflicts_with = "mainnet", + conflicts_with = "testnet", + conflicts_with = "devnet" + )] + pub config_path: Option, } #[derive(Parser, PartialEq, Clone, Debug)] @@ -117,19 +126,26 @@ pub fn main() { match opts.command { Command::Start(cmd) => { - let network = match (cmd.devnet, cmd.testnet, cmd.mainnet) { - (true, false, false) => StacksNetwork::Devnet, - (false, true, false) => StacksNetwork::Testnet, - (false, false, true) => StacksNetwork::Mainnet, + let config = match (cmd.devnet, cmd.testnet, cmd.mainnet, cmd.config_path) { + (true, false, false, _) => Config::devnet_default(), + (false, true, false, _) => Config::testnet_default(), + (false, false, true, _) => Config::mainnet_default(), + (false, false, false, Some(config_path)) => { + match Config::from_file_path(&config_path) { + Ok(config) => config, + Err(e) => { + println!("{e}"); + process::exit(1); + } + } + } _ => { - println!( - "{}", - format_err!("network flag required (devnet, testnet, mainnet)") - ); + println!("network flag required (devnet, testnet, mainnet)"); process::exit(1); } }; - start_node(&network); + println!("{:?}", config); + start_node(config); } Command::Replay(cmd) => { let network = match (cmd.devnet, cmd.testnet, cmd.mainnet) { @@ -191,9 +207,9 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply: let bitcoin_port = bitcoind_rpc_url .port() .expect("unable to retrieve port from bitcoin_url"); - config.indexer.bitcoin_node_rpc_url = format!("http://{}:{}", bitcoin_host, bitcoin_port); - config.indexer.bitcoin_node_rpc_username = bitcoind_rpc_url.username().to_string(); - config.indexer.bitcoin_node_rpc_password = bitcoind_rpc_url + config.network.bitcoin_node_rpc_url = format!("http://{}:{}", bitcoin_host, bitcoin_port); + config.network.bitcoin_node_rpc_username = bitcoind_rpc_url.username().to_string(); + config.network.bitcoin_node_rpc_password = bitcoind_rpc_url .password() .expect("unable to retrieve password from bitcoin_url") .to_string(); @@ -258,9 +274,9 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply: initial_hook_formation: None, ingestion_port: DEFAULT_INGESTION_PORT, control_port: DEFAULT_CONTROL_PORT, - bitcoin_node_username: config.indexer.bitcoin_node_rpc_username.clone(), - bitcoin_node_password: config.indexer.bitcoin_node_rpc_password.clone(), - bitcoin_node_rpc_host: config.indexer.bitcoin_node_rpc_url.clone(), + bitcoin_node_username: config.network.bitcoin_node_rpc_username.clone(), + bitcoin_node_password: config.network.bitcoin_node_rpc_password.clone(), + bitcoin_node_rpc_host: config.network.bitcoin_node_rpc_url.clone(), bitcoin_node_rpc_port: bitcoin_port, stacks_node_rpc_host: "http://localhost".into(), stacks_node_rpc_port: 20443, @@ -296,10 +312,10 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply: }; let auth = Auth::UserPass( - config.indexer.bitcoin_node_rpc_username.clone(), - config.indexer.bitcoin_node_rpc_password.clone(), + config.network.bitcoin_node_rpc_username.clone(), + config.network.bitcoin_node_rpc_password.clone(), ); - let bitcoin_rpc = Client::new(&config.indexer.bitcoin_node_rpc_url, auth).unwrap(); + let bitcoin_rpc = Client::new(&config.network.bitcoin_node_rpc_url, auth).unwrap(); loop { let event = match observer_event_rx.recv() { @@ -504,7 +520,7 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply: }; let block = match bitcoin_rpc.get_block(&block_hash) { - Ok(block) => build_block(block, cursor, &config.indexer), + Ok(block) => build_block(block, cursor, &config.network), Err(e) => { error!("unable to retrieve block hash {}", cursor); continue; @@ -611,7 +627,7 @@ pub fn start_replay_flow(network: &StacksNetwork, bitcoind_rpc_url: Url, apply: } } -pub fn start_node(network: &StacksNetwork) { +pub fn start_node(mut config: Config) { let (digestion_tx, digestion_rx) = channel(); let (observer_event_tx, observer_event_rx) = channel(); let (observer_command_tx, observer_command_rx) = channel(); @@ -625,13 +641,6 @@ pub fn start_node(network: &StacksNetwork) { }) .expect("Error setting Ctrl-C handler"); - let mut config = match network { - StacksNetwork::Devnet => Config::devnet_default(), - StacksNetwork::Testnet => Config::testnet_default(), - StacksNetwork::Mainnet => Config::mainnet_default(), - _ => unreachable!(), - }; - if config.is_initial_ingestion_required() { // Download default tsv. if config.rely_on_remote_tsv() && config.should_download_remote_tsv() { @@ -692,8 +701,8 @@ pub fn start_node(network: &StacksNetwork) { initial_hook_formation: None, ingestion_port: DEFAULT_INGESTION_PORT, control_port: DEFAULT_CONTROL_PORT, - bitcoin_node_username: config.indexer.bitcoin_node_rpc_username.clone(), - bitcoin_node_password: config.indexer.bitcoin_node_rpc_password.clone(), + bitcoin_node_username: config.network.bitcoin_node_rpc_username.clone(), + bitcoin_node_password: config.network.bitcoin_node_rpc_password.clone(), bitcoin_node_rpc_host: "http://localhost".into(), bitcoin_node_rpc_port: 18443, stacks_node_rpc_host: "http://localhost".into(), diff --git a/components/chainhook-node/src/config/file.rs b/components/chainhook-node/src/config/file.rs index 3226da82c..b9674e2bc 100644 --- a/components/chainhook-node/src/config/file.rs +++ b/components/chainhook-node/src/config/file.rs @@ -1,17 +1,18 @@ -#[derive(Clone, Debug)] +#[derive(Deserialize, Debug, Clone)] pub struct ConfigFile { - pub storage: Option, - pub event_source: Vec, - pub chainhooks: Option, + pub storage: StorageConfigFile, + pub event_source: Option>, + pub chainhooks: ChainhooksConfigFile, + pub network: NetworkConfigFile, } -#[derive(Clone, Debug)] +#[derive(Deserialize, Debug, Clone)] pub struct StorageConfigFile { - pub driver: Option, - pub redis_uri: Option, + pub driver: String, + pub redis_uri: String, } -#[derive(Clone, Debug)] +#[derive(Deserialize, Debug, Clone)] pub struct EventSourceConfigFile { pub source_type: Option, pub stacks_node_url: Option, @@ -21,8 +22,17 @@ pub struct EventSourceConfigFile { pub tsv_file_url: Option, } -#[derive(Clone, Debug)] +#[derive(Deserialize, Debug, Clone)] pub struct ChainhooksConfigFile { pub max_stacks_registrations: Option, pub max_bitcoin_registrations: Option, } + +#[derive(Deserialize, Debug, Clone)] +pub struct NetworkConfigFile { + pub mode: String, + pub bitcoin_node_rpc_url: String, + pub bitcoin_node_rpc_username: String, + pub bitcoin_node_rpc_password: String, + pub stacks_node_rpc_url: String, +} diff --git a/components/chainhook-node/src/config/mod.rs b/components/chainhook-node/src/config/mod.rs index 480f0a55f..7e438567b 100644 --- a/components/chainhook-node/src/config/mod.rs +++ b/components/chainhook-node/src/config/mod.rs @@ -3,17 +3,19 @@ pub mod file; pub use chainhook_event_observer::indexer::IndexerConfig; use chainhook_types::{BitcoinNetwork, StacksNetwork}; pub use file::ConfigFile; +use std::fs::File; +use std::io::{BufReader, Read}; use std::path::PathBuf; -const DEFAULT_MAINNET_TSV_ARCHIVE: &str = "https://storage.googleapis.com/hirosystems-archive/mainnet/api/mainnet-blockchain-api-5.0.3-latest.tar.gz"; -const DEFAULT_TESTNET_TSV_ARCHIVE: &str = "https://storage.googleapis.com/hirosystems-archive/testnet/api/testnet-blockchain-api-5.0.3-latest.tar.gz"; +const DEFAULT_MAINNET_TSV_ARCHIVE: &str = "https://storage.googleapis.com/hirosystems-archive/mainnet/api/mainnet-blockchain-api-latest.tar.gz"; +const DEFAULT_TESTNET_TSV_ARCHIVE: &str = "https://storage.googleapis.com/hirosystems-archive/testnet/api/testnet-blockchain-api-latest.tar.gz"; #[derive(Clone, Debug)] pub struct Config { pub storage: StorageConfig, pub event_sources: Vec, pub chainhooks: ChainhooksConfig, - pub indexer: IndexerConfig, + pub network: IndexerConfig, } #[derive(Clone, Debug)] @@ -61,6 +63,70 @@ pub struct ChainhooksConfig { } impl Config { + pub fn from_file_path(file_path: &str) -> Result { + let file = File::open(file_path) + .map_err(|e| format!("unable to read file {}\n{:?}", file_path, e))?; + let mut file_reader = BufReader::new(file); + let mut file_buffer = vec![]; + file_reader + .read_to_end(&mut file_buffer) + .map_err(|e| format!("unable to read file {}\n{:?}", file_path, e))?; + + let config_file: ConfigFile = match toml::from_slice(&file_buffer) { + Ok(s) => s, + Err(e) => { + return Err(format!("Config file malformatted {}", e.to_string())); + } + }; + Config::from_config_file(config_file) + } + + pub fn from_config_file(config_file: ConfigFile) -> Result { + let (stacks_network, bitcoin_network) = match config_file.network.mode.as_str() { + "devnet" => (StacksNetwork::Devnet, BitcoinNetwork::Regtest), + "testnet" => (StacksNetwork::Testnet, BitcoinNetwork::Testnet), + "mainnet" => (StacksNetwork::Mainnet, BitcoinNetwork::Mainnet), + _ => return Err("network.mode not supported".to_string()), + }; + + let config = Config { + storage: StorageConfig { + driver: StorageDriver::Redis(RedisConfig { + uri: config_file.storage.redis_uri.to_string(), + }), + cache_path: "cache".into(), + }, + event_sources: vec![EventSourceConfig::StacksNode(StacksNodeConfig { + host: config_file.network.stacks_node_rpc_url.to_string(), + })], + chainhooks: ChainhooksConfig { + max_stacks_registrations: config_file + .chainhooks + .max_stacks_registrations + .unwrap_or(100), + max_bitcoin_registrations: config_file + .chainhooks + .max_bitcoin_registrations + .unwrap_or(100), + }, + network: IndexerConfig { + stacks_node_rpc_url: config_file.network.stacks_node_rpc_url.to_string(), + bitcoin_node_rpc_url: config_file.network.bitcoin_node_rpc_url.to_string(), + bitcoin_node_rpc_username: config_file + .network + .bitcoin_node_rpc_username + .to_string(), + bitcoin_node_rpc_password: config_file + .network + .bitcoin_node_rpc_password + .to_string(), + stacks_network, + bitcoin_network, + }, + }; + Ok(config) + } + pub fn is_initial_ingestion_required(&self) -> bool { for source in self.event_sources.iter() { match source { @@ -155,7 +221,7 @@ impl Config { max_stacks_registrations: 50, max_bitcoin_registrations: 50, }, - indexer: IndexerConfig { + network: IndexerConfig { stacks_node_rpc_url: "http://0.0.0.0:20443".into(), bitcoin_node_rpc_url: "http://0.0.0.0:18443".into(), bitcoin_node_rpc_username: "devnet".into(), @@ -181,7 +247,7 @@ impl Config { max_stacks_registrations: 10, max_bitcoin_registrations: 10, }, - indexer: IndexerConfig { + network: IndexerConfig { stacks_node_rpc_url: "http://0.0.0.0:20443".into(), bitcoin_node_rpc_url: "http://0.0.0.0:18443".into(), bitcoin_node_rpc_username: "devnet".into(), @@ -207,7 +273,7 @@ impl Config { max_stacks_registrations: 10, max_bitcoin_registrations: 10, }, - indexer: IndexerConfig { + network: IndexerConfig { stacks_node_rpc_url: "http://0.0.0.0:20443".into(), bitcoin_node_rpc_url: "http://0.0.0.0:18443".into(), bitcoin_node_rpc_username: "devnet".into(), diff --git a/components/chainhook-node/src/lib.rs b/components/chainhook-node/src/lib.rs index f494bcdae..f18ca2829 100644 --- a/components/chainhook-node/src/lib.rs +++ b/components/chainhook-node/src/lib.rs @@ -4,8 +4,10 @@ extern crate serde_json; #[macro_use] extern crate slog_scope; -extern crate serde; +#[macro_use] extern crate serde_derive; +extern crate serde; + pub mod block; pub mod config; diff --git a/components/chainhook-node/src/main.rs b/components/chainhook-node/src/main.rs index ec6484616..b5dc48a60 100644 --- a/components/chainhook-node/src/main.rs +++ b/components/chainhook-node/src/main.rs @@ -7,8 +7,10 @@ extern crate slog_scope; #[macro_use] extern crate hiro_system_kit; -extern crate serde; +#[macro_use] extern crate serde_derive; + +extern crate serde; extern crate slog; use slog_async;