From 959818a7cead6f63e251710ebf7576ef6df3ef46 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 23 Jan 2025 18:02:23 +0100 Subject: [PATCH] inline services-mango-lib --- Cargo.lock | 9 ++- bin/comparer/Cargo.toml | 2 +- bin/comparer/src/config.rs | 4 +- bin/comparer/src/persister.rs | 4 +- bin/indexer/Cargo.toml | 2 +- bin/indexer/src/config.rs | 2 +- bin/indexer/src/persister.rs | 4 +- lib/services-lib/Cargo.toml | 21 +++++++ lib/services-lib/src/env_helper.rs | 21 +++++++ lib/services-lib/src/lib.rs | 3 + .../src/postgres_configuration.rs | 18 ++++++ lib/services-lib/src/postgres_connection.rs | 60 +++++++++++++++++++ 12 files changed, 136 insertions(+), 14 deletions(-) create mode 100644 lib/services-lib/Cargo.toml create mode 100644 lib/services-lib/src/env_helper.rs create mode 100644 lib/services-lib/src/lib.rs create mode 100644 lib/services-lib/src/postgres_configuration.rs create mode 100644 lib/services-lib/src/postgres_connection.rs diff --git a/Cargo.lock b/Cargo.lock index 8687095..d8b07d6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1596,7 +1596,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "services-mango-lib", + "services-lib", "sha2 0.10.8", "shellexpand 2.1.2", "solana-account-decoder", @@ -3499,7 +3499,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "services-mango-lib", + "services-lib", "sha2 0.10.8", "shellexpand 2.1.2", "solana-account-decoder", @@ -6714,9 +6714,8 @@ dependencies = [ ] [[package]] -name = "services-mango-lib" -version = "0.1.0" -source = "git+https://github.com/blockworks-foundation/mango-v4.git#905cc0141467600ddac6c22134e2126332b14c78" +name = "services-lib" +version = "0.0.1" dependencies = [ "anyhow", "base64 0.21.7", diff --git a/bin/comparer/Cargo.toml b/bin/comparer/Cargo.toml index 09bbc0f..7975bd3 100644 --- a/bin/comparer/Cargo.toml +++ b/bin/comparer/Cargo.toml @@ -47,7 +47,7 @@ router-config-lib = { path = "../../lib/router-config-lib" } router-lib = { path = "../../lib/router-lib/", version = "0.0.1" } base64 = "0.21.7" bincode = "1.3.3" -services-mango-lib = { git = "https://github.com/blockworks-foundation/mango-v4.git" } +services-lib = { path = "../../lib/services-lib" } tokio-postgres = { version = "0.7", features = ["with-chrono-0_4"] } tokio-postgres-rustls = "0.9.0" postgres_query = { git = "https://github.com/nolanderc/rust-postgres-query", rev = "b4422051c8a31fbba4a35f88004c1cefb1878dd5" } diff --git a/bin/comparer/src/config.rs b/bin/comparer/src/config.rs index 144eb23..7d5b3e0 100644 --- a/bin/comparer/src/config.rs +++ b/bin/comparer/src/config.rs @@ -1,5 +1,5 @@ -use services_mango_lib::env_helper::string_or_env as serde_string_or_env; -use services_mango_lib::postgres_configuration::PostgresConfiguration; +use services_lib::env_helper::string_or_env as serde_string_or_env; +use services_lib::postgres_configuration::PostgresConfiguration; #[derive(Clone, Debug, Default, serde_derive::Deserialize)] pub struct Config { diff --git a/bin/comparer/src/persister.rs b/bin/comparer/src/persister.rs index 18f94e7..a429953 100644 --- a/bin/comparer/src/persister.rs +++ b/bin/comparer/src/persister.rs @@ -1,7 +1,7 @@ use crate::config::Config; use async_channel::Receiver; -use services_mango_lib::postgres_configuration::PostgresConfiguration; -use services_mango_lib::postgres_connection; +use services_lib::postgres_configuration::PostgresConfiguration; +use services_lib::postgres_connection; use solana_program::pubkey::Pubkey; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; diff --git a/bin/indexer/Cargo.toml b/bin/indexer/Cargo.toml index bbcdf69..4bff577 100644 --- a/bin/indexer/Cargo.toml +++ b/bin/indexer/Cargo.toml @@ -46,7 +46,7 @@ router-feed-lib = { path = "../../lib/router-feed-lib" } router-config-lib = { path = "../../lib/router-config-lib" } base64 = "0.21.7" bincode = "1.3.3" -services-mango-lib = { git = "https://github.com/blockworks-foundation/mango-v4.git" } +services-lib = { path = "../../lib/services-lib" } tokio-postgres = { version = "0.7", features = ["with-chrono-0_4"] } tokio-postgres-rustls = "0.9.0" postgres_query = { git = "https://github.com/nolanderc/rust-postgres-query", rev = "b4422051c8a31fbba4a35f88004c1cefb1878dd5" } diff --git a/bin/indexer/src/config.rs b/bin/indexer/src/config.rs index be56a35..cbdc8a5 100644 --- a/bin/indexer/src/config.rs +++ b/bin/indexer/src/config.rs @@ -1,5 +1,5 @@ use router_config_lib::AccountDataSourceConfig; -use services_mango_lib::postgres_configuration::PostgresConfiguration; +use services_lib::postgres_configuration::PostgresConfiguration; #[derive(Clone, Debug, Default, serde_derive::Deserialize)] pub struct Config { diff --git a/bin/indexer/src/persister.rs b/bin/indexer/src/persister.rs index 5bbf847..88b516f 100644 --- a/bin/indexer/src/persister.rs +++ b/bin/indexer/src/persister.rs @@ -1,7 +1,7 @@ use crate::config::MetricsConfig; use async_channel::Receiver; -use services_mango_lib::postgres_configuration::PostgresConfiguration; -use services_mango_lib::postgres_connection; +use services_lib::postgres_configuration::PostgresConfiguration; +use services_lib::postgres_connection; use solana_sdk::signature::Signature; use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; diff --git a/lib/services-lib/Cargo.toml b/lib/services-lib/Cargo.toml new file mode 100644 index 0000000..fb8d30a --- /dev/null +++ b/lib/services-lib/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "services-lib" +version = "0.0.1" +edition = "2021" + +[lib] +doctest = false + +[dependencies] +anyhow = "1.0" +base64 = "0.21" +tokio = { version = "1", features = ["full"] } +tokio-postgres = { version = "0.7", features = ["with-chrono-0_4"] } +tokio-postgres-rustls = "0.9.0" +postgres-types = { version = "0.2", features = ["array-impls", "derive", "with-chrono-0_4"] } +postgres-native-tls = "0.5" +native-tls = "0.2" +rustls = "0.20.8" +postgres_query = { git = "https://github.com/nolanderc/rust-postgres-query", rev = "b4422051c8a31fbba4a35f88004c1cefb1878dd5" } +tracing = { version = "0.1", features = ["log"] } +serde = { version = "1.0.188", features = ["derive"] } diff --git a/lib/services-lib/src/env_helper.rs b/lib/services-lib/src/env_helper.rs new file mode 100644 index 0000000..242262a --- /dev/null +++ b/lib/services-lib/src/env_helper.rs @@ -0,0 +1,21 @@ +use serde::{Deserialize, Deserializer}; +use std::env; + +/// Get a string content, or the content of an Env variable it the string start with $ +/// +/// Example: +/// - "abc" -> "abc" +/// - "$something" -> read env variable named something and return it's content +/// +/// *WARNING*: May kill the program if we are asking for anv environment variable that does not exist +pub fn string_or_env<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let value_or_env = String::deserialize(deserializer)?; + let value = match &value_or_env.chars().next().unwrap() { + '$' => env::var(&value_or_env[1..]).expect("reading from env"), + _ => value_or_env, + }; + Ok(value) +} diff --git a/lib/services-lib/src/lib.rs b/lib/services-lib/src/lib.rs new file mode 100644 index 0000000..77b5705 --- /dev/null +++ b/lib/services-lib/src/lib.rs @@ -0,0 +1,3 @@ +pub mod env_helper; +pub mod postgres_configuration; +pub mod postgres_connection; diff --git a/lib/services-lib/src/postgres_configuration.rs b/lib/services-lib/src/postgres_configuration.rs new file mode 100644 index 0000000..718f3a5 --- /dev/null +++ b/lib/services-lib/src/postgres_configuration.rs @@ -0,0 +1,18 @@ +use crate::env_helper::string_or_env; +use serde::Deserialize; + +#[derive(Clone, Debug, Deserialize, Default)] +pub struct PostgresConfiguration { + #[serde(deserialize_with = "string_or_env")] + pub connection_string: String, + pub allow_invalid_certs: bool, + pub tls: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct PostgresTlsConfig { + /// CA Cert file or env var + pub ca_cert_path: String, + /// PKCS12 client cert path + pub client_key_path: String, +} diff --git a/lib/services-lib/src/postgres_connection.rs b/lib/services-lib/src/postgres_connection.rs new file mode 100644 index 0000000..bc393ac --- /dev/null +++ b/lib/services-lib/src/postgres_connection.rs @@ -0,0 +1,60 @@ +use crate::postgres_configuration::PostgresConfiguration; +use native_tls::{Certificate, Identity, TlsConnector}; +use postgres_native_tls::MakeTlsConnector; +use std::{env, fs}; +use tokio::task::JoinHandle; +use tokio_postgres::Client; + +pub async fn connect( + config: &PostgresConfiguration, +) -> anyhow::Result<(Client, JoinHandle>)> { + // openssl pkcs12 -export -in client.cer -inkey client-key.cer -out client.pks + // base64 -i ca.cer -o ca.cer.b64 && base64 -i client.pks -o client.pks.b64 + // fly secrets set PG_CA_CERT=- < ./ca.cer.b64 -a mango-fills + // fly secrets set PG_CLIENT_KEY=- < ./client.pks.b64 -a mango-fills + let tls = match &config.tls { + Some(tls) => { + use base64::{engine::general_purpose, Engine as _}; + let ca_cert = match &tls.ca_cert_path.chars().next().unwrap() { + '$' => general_purpose::STANDARD + .decode( + env::var(&tls.ca_cert_path[1..]) + .expect("reading client cert from env") + .into_bytes(), + ) + .expect("decoding client cert"), + _ => fs::read(&tls.ca_cert_path).expect("reading client cert from file"), + }; + let client_key = match &tls.client_key_path.chars().next().unwrap() { + '$' => general_purpose::STANDARD + .decode( + env::var(&tls.client_key_path[1..]) + .expect("reading client key from env") + .into_bytes(), + ) + .expect("decoding client key"), + _ => fs::read(&tls.client_key_path).expect("reading client key from file"), + }; + MakeTlsConnector::new( + TlsConnector::builder() + .add_root_certificate(Certificate::from_pem(&ca_cert)?) + .identity(Identity::from_pkcs12(&client_key, "pass")?) + .danger_accept_invalid_certs(config.allow_invalid_certs) + .build()?, + ) + } + None => MakeTlsConnector::new( + TlsConnector::builder() + .danger_accept_invalid_certs(config.allow_invalid_certs) + .build()?, + ), + }; + + let config = config.clone(); + + let (client, connection) = tokio_postgres::connect(&config.connection_string, tls).await?; + + let handle = tokio::spawn(async move { connection.await }); + + Ok((client, handle)) +}