Merge pull request #2 from ABlockOfficial/qa

Add Logging; Improve API request handling

.github/pull_request_template.md

@@ -0,0 +1,39 @@
+## Description
+Provide a clear and concise description of the changes introduced in this pull request.
+
+Fixes # (issue number)
+
+## Changelog
+
+- List the changes to the codebase that this PR introduces
+
+## Type of Change
+Please mark the appropriate option by putting an "x" inside the brackets:
+
+- [ ] Bug fix
+- [ ] New feature
+- [ ] Enhancement or optimization
+- [ ] Documentation update
+- [ ] Other (please specify)
+
+## Checklist
+Put an "x" in the boxes that apply. If you're unsure about any of these, don't hesitate to ask. We're here to help!
+
+- [ ] I have tested the changes locally and they work as expected.
+- [ ] I have added necessary documentation or updated existing documentation.
+- [ ] My code follows the project's coding standards and style guidelines.
+- [ ] I have added/updated relevant tests to ensure the changes are properly covered.
+- [ ] I have checked for and resolved any merge conflicts.
+- [ ] My commits have clear and descriptive messages.
+
+## Screenshots (if applicable)
+If the changes affect the UI or have visual effects, please provide screenshots or GIFs showcasing the changes.
+
+## Additional Context (if applicable)
+Add any additional context or information about the changes that may be helpful in understanding the pull request.
+
+## Related Issues (if applicable)
+If this pull request is related to any existing issues, please list them here.
+
+## Requested Reviewers
+Mention any specific individuals or teams you would like to request a review from.

.github/workflows/rust.yml

@@ -0,0 +1,22 @@
+name: Rust
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build
+      run: cargo build --verbose
+    - name: Run tests
+      run: cargo test --verbose

Cargo.toml

@@ -20,6 +20,8 @@ serde = { version = "1.0.173", features = ["derive"] }
 serde_json = "1.0.103"
 tokio = { version="1.29.1", features=["full"] }
 tracing = "0.1.37"
+tracing-subscriber = "0.3.17"
+tracing-futures = "0.2.3"
 warp = "0.3.5"
-valence_core = "0.1.0" 
+valence_core = "0.1.2"
 valence_market = "0.1.0"

config.toml

@@ -1,12 +1,14 @@
 # This is a sample config TOML for Valence
 debug = false
 extern_port = 3030
-cache_url = "redis://cache"
+cache_url = "redis://0.0.0.0"
 cache_port = "6379"
 cache_password = ""
-db_url = "mongodb://db"
+db_protocol = "mongodb://"
+db_url = "0.0.0.0"
 db_port = "27017"
-db_password = ""
+db_user = "root"
+db_password = "example"
 body_limit = 4096
 cache_ttl = 600 # cache lifetime in seconds
 

src/api/handlers.rs

@@ -1,11 +1,13 @@
 use crate::interfaces::SetRequestData;
 use futures::lock::Mutex;
+use serde_json::Value;
 use std::sync::Arc;
+use tracing::{error, info, warn};
 use valence_core::api::errors::ApiErrorType;
 use valence_core::api::interfaces::CFilterConnection;
 use valence_core::api::responses::{json_serialize_embed, CallResponse, JsonReply};
 use valence_core::db::handler::{CacheHandler, KvStoreConnection};
-use valence_core::utils::{deserialize_data, serialize_data};
+use valence_core::utils::serialize_data;
 
 // ========= BASE HANDLERS ========= //
 
@@ -27,43 +29,44 @@ pub async fn get_data_handler<
     c_filter: CFilterConnection,
 ) -> Result<JsonReply, JsonReply> {
     let r = CallResponse::new("get_data");
+    info!("GET_DATA requested with headers: {:?}", headers);
+
     let address = headers
         .get("address")
         .and_then(|n| n.to_str().ok())
         .unwrap_or_default();
 
     // Check if address is in cuckoo filter
     if !c_filter.lock().await.contains(&address) {
+        error!("Address not found in cuckoo filter");
         return r.into_err_internal(ApiErrorType::CuckooFilterLookupFailed);
     }
 
     // Check cache first
-    let cache_result = cache.lock().await.get_data(&address).await;
+    let cache_result: Result<Option<Value>, _> = cache.lock().await.get_data(address).await;
 
     match cache_result {
         Ok(value) => {
-            // Return data from cache
-            let final_data = if value.is_some() {
-                deserialize_data::<String>(value.unwrap())
-            } else {
-                "".to_string()
-            };
+            let value_stable = value.unwrap_or_default();
+
             r.into_ok(
                 "Data retrieved successfully",
-                json_serialize_embed(final_data),
+                json_serialize_embed(value_stable),
             )
         }
         Err(_) => {
+            warn!("Cache lookup failed for address: {}", address);
+            warn!("Attempting to retrieve data from DB");
+
             // Get data from DB
-            let db_result = db.lock().await.get_data(&address).await;
+            let db_result: Result<Option<Value>, _> = db.lock().await.get_data(address).await;
 
             match db_result {
                 Ok(value) => {
-                    // Return data from DB
-                    let final_data = deserialize_data::<String>(value.unwrap());
+                    let value_stable = value.unwrap_or_default();
                     r.into_ok(
                         "Data retrieved successfully",
-                        json_serialize_embed(final_data),
+                        json_serialize_embed(value_stable),
                     )
                 }
                 Err(_) => r.into_err_internal(ApiErrorType::DBInsertionFailed),
@@ -91,6 +94,7 @@ pub async fn set_data_handler<
     cache_ttl: usize,
 ) -> Result<JsonReply, JsonReply> {
     let r = CallResponse::new("set_data");
+    info!("SET_DATA requested with payload: {:?}", payload);
 
     // Add to cache
     let cache_result = cache
@@ -109,15 +113,10 @@ pub async fn set_data_handler<
                 .expire_entry(&payload.address, cache_ttl)
                 .await;
 
-            // Set data in DB
-            let data = match serde_json::from_str::<serde_json::Value>(&payload.data) {
-                Ok(data) => data,
-                Err(_) => {
-                    return r.into_err_internal(ApiErrorType::DataSerializationFailed);
-                }
-            };
-
-            db.lock().await.set_data(&payload.address, data).await
+            db.lock()
+                .await
+                .set_data(&payload.address, payload.data)
+                .await
         }
         Err(_) => {
             return r.into_err_internal(ApiErrorType::CacheInsertionFailed);
@@ -134,7 +133,7 @@ pub async fn set_data_handler<
 
     match c_filter_result {
         Ok(_) => r.into_ok(
-            "Data set succcessfully",
+            "Data set successfully",
             json_serialize_embed(payload.address),
         ),
         Err(_) => r.into_err_internal(ApiErrorType::CuckooFilterInsertionFailed),

src/api/routes.rs

@@ -1,6 +1,7 @@
 use crate::api::handlers::{get_data_handler, set_data_handler};
 use futures::lock::Mutex;
 use std::sync::Arc;
+use tracing::debug;
 use valence_core::api::interfaces::CFilterConnection;
 use valence_core::api::utils::{
     get_cors, map_api_res, post_cors, sig_verify_middleware, with_node_component,
@@ -27,6 +28,8 @@ pub fn get_data<
     cache: Arc<Mutex<C>>,
     cuckoo_filter: CFilterConnection,
 ) -> impl Filter<Extract = (impl Reply,), Error = Rejection> + Clone {
+    debug!("Setting up get_data route");
+
     warp::path("get_data")
         .and(warp::get())
         .and(sig_verify_middleware())
@@ -35,6 +38,7 @@ pub fn get_data<
         .and(with_node_component(db))
         .and(with_node_component(cuckoo_filter))
         .and_then(move |_, headers, cache, db, cf| {
+            debug!("GET_DATA requested");
             map_api_res(get_data_handler(headers, db, cache, cf))
         })
         .with(get_cors())
@@ -60,6 +64,8 @@ pub fn set_data<
     body_limit: u64,
     cache_ttl: usize,
 ) -> impl Filter<Extract = (impl Reply,), Error = Rejection> + Clone {
+    debug!("Setting up set_data route");
+
     warp::path("set_data")
         .and(warp::post())
         .and(sig_verify_middleware())
@@ -70,6 +76,7 @@ pub fn set_data<
         .and(with_node_component(cuckoo_filter))
         .and(with_node_component(cache_ttl))
         .and_then(move |_, info, cache, db, cf, cttl| {
+            debug!("SET_DATA requested");
             map_api_res(set_data_handler(info, db, cache, cf, cttl))
         })
         .with(post_cors())

src/constants.rs

@@ -3,9 +3,11 @@
 pub const CONFIG_FILE: &str = "config.toml";
 pub const SETTINGS_DEBUG: bool = false;
 pub const SETTINGS_EXTERN_PORT: u16 = 8080;
-pub const SETTINGS_DB_URL: &str = "mongodb://127.0.0.1";
+pub const SETTINGS_DB_PROTOCOL: &str = "mongodb";
+pub const SETTINGS_DB_URL: &str = "127.0.0.1";
 pub const SETTINGS_DB_PORT: &str = "12701";
-pub const SETTINGS_DB_PASSWORD: &str = "password";
+pub const SETTINGS_DB_USER: &str = "root";
+pub const SETTINGS_DB_PASSWORD: &str = "example";
 pub const SETTINGS_CACHE_URL: &str = "redis://127.0.0.1";
 pub const SETTINGS_CACHE_PORT: &str = "6379";
 pub const SETTINGS_CACHE_PASSWORD: &str = "password";

src/interfaces.rs

@@ -1,4 +1,5 @@
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 
 // Define a struct to hold the data (public key, address, signature)
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -9,12 +10,14 @@ pub struct GetRequestData {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SetRequestData {
     pub address: String,
-    pub data: String,
+    pub data: Value,
 }
 
 pub struct EnvConfig {
     pub debug: bool,
     pub extern_port: u16,
+    pub db_protocol: String,
+    pub db_user: String,
     pub db_url: String,
     pub db_port: String,
     pub db_password: String,

src/main.rs

@@ -11,19 +11,25 @@ use crate::api::routes::*;
 use crate::utils::{construct_mongodb_conn, construct_redis_conn, load_config, print_welcome};
 use futures::lock::Mutex;
 use std::sync::Arc;
+use tracing::info;
 use valence_core::api::utils::handle_rejection;
-use valence_market::api::routes::*;
+
 use warp::Filter;
 
 #[tokio::main]
 async fn main() {
+    tracing_subscriber::fmt::init();
+
     let config = load_config();
     let cache_addr = format!("{}:{}", config.cache_url, config.cache_port);
-    let db_addr = format!("{}:{}", config.db_url, config.db_port);
+    let db_addr = format!(
+        "{}{}:{}@{}:{}",
+        config.db_protocol, config.db_user, config.db_password, config.db_url, config.db_port
+    );
     let cuckoo_filter = Arc::new(Mutex::new(cuckoofilter::CuckooFilter::new()));
 
-    println!("Connecting to Redis at {}", cache_addr);
-    println!("Connecting to MongoDB at {}", db_addr);
+    info!("Connecting to Redis at {}", cache_addr);
+    info!("Connecting to MongoDB at {}", db_addr);
 
     let cache_conn = construct_redis_conn(&cache_addr).await;
     let db_conn = construct_mongodb_conn(&db_addr).await;
@@ -58,7 +64,7 @@ async fn main() {
 
     print_welcome(&db_addr, &cache_addr);
 
-    println!("Server running at localhost:{}", config.extern_port);
+    info!("Server running at localhost:{}", config.extern_port);
 
     warp::serve(routes)
         .run(([0, 0, 0, 0], config.extern_port))

src/tests/constants.rs

@@ -1,4 +1,4 @@
-pub const TEST_VALID_ADDRESS: &'static str = "Hello World!";
-pub const TEST_VALID_PUB_KEY: &'static str =
+pub const TEST_VALID_ADDRESS: &str = "Hello World!";
+pub const TEST_VALID_PUB_KEY: &str =
     "a33118ddaa685e7feb1f89168740fa2b0904b899b719d36b39f62ee9283e9455";
-pub const TEST_VALID_SIG: &'static str = "55772ac82f1968f7b04597a0a33f5fbbfce1a05f405c491da0ed758f0f8fbd63a15c9c629a413fa6b9a51fb5d6ccd9eb1514b566a75f5f5b265bd80dae4b440c";
+pub const TEST_VALID_SIG: &str = "55772ac82f1968f7b04597a0a33f5fbbfce1a05f405c491da0ed758f0f8fbd63a15c9c629a413fa6b9a51fb5d6ccd9eb1514b566a75f5f5b265bd80dae4b440c";

src/tests/mod.rs

@@ -86,7 +86,7 @@ async fn test_get_data() {
     assert_eq!(res.status(), 200);
     assert_eq!(
         res.body(),
-        "{\"status\":\"Success\",\"reason\":\"Data retrieved successfully\",\"route\":\"get_data\",\"content\":\"\"}"
+        "{\"status\":\"Success\",\"reason\":\"Data retrieved successfully\",\"route\":\"get_data\",\"content\":\"{\\\"Hello\\\":20}\"}"
     );
 }
 
@@ -122,6 +122,6 @@ async fn test_set_data() {
     assert_eq!(res.status(), 200);
     assert_eq!(
         res.body(),
-        "{\"status\":\"Success\",\"reason\":\"Data set succcessfully\",\"route\":\"set_data\",\"content\":\"0x123\"}"
+        "{\"status\":\"Success\",\"reason\":\"Data set successfully\",\"route\":\"set_data\",\"content\":\"0x123\"}"
     );
 }