Merge pull request #11 from AIBlockOfficial/feature_delete

Feature delete - `del_data` route

Cargo.toml

@@ -23,4 +23,4 @@ tracing = "0.1.37"
 tracing-subscriber = "0.3.17"
 tracing-futures = "0.2.3"
 warp = "0.3.5"
-valence_core = "0.1.7"
+valence_core = "0.1.8"

README.md

@@ -33,8 +33,9 @@
       <a href="#getting-started">Getting Started</a>
       <ul>
         <li><a href="#prerequisites">Prerequisites</a></li>
-        <li><a href="#running-the-server">Running The Server</a></li>
-        </ul>
+        <li><a href="#installation">Installation</a></li>
+        <li><a href="#running-the-server">Running the server</a></li>
+      </ul>
     </li>
     <li>
       <a href="#how-it-works">How it Works</a>
@@ -44,6 +45,7 @@
             <ul>
                 <li><a href="#set_data">set_data</a></li>
                 <li><a href="#get_data">get_data</a></li>
+                <li><a href="#del_data">del_data</a></li>
             </ul>
         </li>
         <li><a href="#further-work">Further Work</a></li>
@@ -89,7 +91,7 @@ docker build -t valence .
 
 ..
 
-### 🏎️ Running
+### 🏎️ Running the server
 
 To use the server as is, you can simply run the following in the root folder of the repo:
 
@@ -133,14 +135,17 @@ Sets data in the Redis instance and marks it for pending retrieval in the server
 }
 ```
 
-The body of the `set_data` call would contain the data being exchanged:
+The body of the `set_data` call would contain the `value_id` for that entry and the `data` being exchanged :
 
 ```json
 {
+    "data_id": "EntryId"
     "data": "hello Bob"
 }
 ```
 
+`data_id` is required and allows for mutiple entries under one address. If the `data_id` value is the same as an existing entry for that address, it is updated. If the `data_id` is unique it will be added to the hashmap for that address
+
 The headers that Alice sends in her call will be validated by the Valence, after which they'll be stored at Bob's address for his later retrieval using the `get_data` call.
 
 ..
@@ -152,12 +157,31 @@ Gets pending data from the server for a given address. To retrieve data for Bob,
 [
     {
         "address": "76e...dd6",     // Bob's public key address
+        "public_key": "a4c...e45"   // Bob's public key corresponding to his address
         "signature": "b9f...506",   // Bob's signature of the public key
+    }
+]
+```
+
+If `data_id` is provided in the request (`get_data/[value_id]`), the specific entry associated to that id is retrieved. If no `data_id` is provided, the full hashmap is retrieved.
+
+Again, the Valence will validate the signature before returning the data to Bob.
+
+##### **<img src="https://img.shields.io/badge/DEL-FF0000" alt="DEL"/> `del_data`**
+Delete pending data from the server for a given address. To delete data for Bob, he only has to supply his credentials in the call header:
+
+```json
+[
+    {
+        "address": "76e...dd6",     // Bob's public key address
         "public_key": "a4c...e45"   // Bob's public key corresponding to his address
+        "signature": "b9f...506",   // Bob's signature of the public key
     }
 ]
 ```
 
+If `data_id` is provided in the request (`del_data/[value_id]`), the specific entry associated to that id is deleted. If no `data_id` is provided, the full hashmap is deleted.
+
 Again, the Valence will validate the signature before returning the data to Bob.
 
 **For best practice, it's recommended that Alice and Bob encrypt their data using their private keys, before exchanging it with each other.** This ensures that the data exchange is E2E encrypted, and that the Valence maintains no knowledge of the data's content.
@@ -171,7 +195,7 @@ Again, the Valence will validate the signature before returning the data to Bob.
 - [x] Match public key to address for `get_data` (resolved by using address directly for retrieval)
 - [ ] Add a rate limiting mechanism
 - [x] Set Redis keys to expire (handle cache lifetimes)
-- [ ] Handle multiple data entries per address
+- [x] Handle multiple data entries per address
 - [ ] Add tests
 
 <p align="left">(<a href="#top">back to top</a>)</p>

src/api/handlers.rs

@@ -1,11 +1,11 @@
-use crate::api::utils::{retrieve_from_db, serialize_all_entries};
+use crate::api::utils::{retrieve_from_db, delete_from_db, serialize_all_entries};
 use crate::interfaces::{SetRequestData, SetSaveData};
 use crate::db::handler::{CacheHandler, KvStoreConnection};
 use futures::lock::Mutex;
 use serde_json::Value;
 use std::collections::HashMap;
 use std::sync::Arc;
-use tracing::{error, info, warn};
+use tracing::{error, info, debug};
 use valence_core::api::errors::ApiErrorType;
 use valence_core::api::interfaces::CFilterConnection;
 use valence_core::api::responses::{json_serialize_embed, CallResponse, JsonReply};
@@ -17,7 +17,8 @@ use valence_core::utils::serialize_data;
 ///
 /// ### Arguments
 ///
-/// * `payload` - Request payload
+/// * `headers` - Request headers
+/// * `value_id` - Value ID to retrieve (Optional, if not provided, all values for the address are retrieved)
 /// * `db` - Database connection
 /// * `cache` - Cache connection
 /// * `c_filter` - Cuckoo filter connection
@@ -41,7 +42,7 @@ pub async fn get_data_handler<
 
     // Check if address is in cuckoo filter
     if !c_filter.lock().await.contains(&address) {
-        error!("Address not found in cuckoo filter");
+        error!("{}", ApiErrorType::CuckooFilterLookupFailed );
         return r.into_err_internal(ApiErrorType::CuckooFilterLookupFailed);
     }
 
@@ -51,18 +52,14 @@ pub async fn get_data_handler<
         .get_data::<String>(address, value_id.as_deref())
         .await;
 
-    info!("Cache result: {:?}", cache_result);
-
     match cache_result {
         Ok(value) => {
             match value {
                 Some(value) => {
                     info!("Data retrieved from cache");
                     if let Some(id) = value_id {
                         if !value.contains_key(&id) {
-                            return r.into_err_internal(ApiErrorType::Generic(
-                                "Value ID not found".to_string(),
-                            ));
+                            return r.into_err_internal(ApiErrorType::ValueIdNotFound);
                         }
 
                         let data = value.get(&id).unwrap().clone();
@@ -82,15 +79,13 @@ pub async fn get_data_handler<
                 }
                 None => {
                     // Default to checking from DB if cache is empty
-                    warn!("Cache lookup failed for address: {}", address);
+                    debug!("Cache lookup failed for address: {}, attempting to retrieve data from DB", address);
                     retrieve_from_db(db, address, value_id.as_deref()).await
                 }
             }
         }
         Err(_) => {
-            warn!("Cache lookup failed for address: {}", address);
-            warn!("Attempting to retrieve data from DB");
-
+            debug!("Attempting to retrieve data from DB");
             // Get data from DB
             retrieve_from_db(db, address, value_id.as_deref()).await
         }
@@ -105,6 +100,7 @@ pub async fn get_data_handler<
 /// * `db` - Database connection
 /// * `cache` - Cache connection
 /// * `c_filter` - Cuckoo filter connection
+/// * `cache_ttl` - Cache TTL
 pub async fn set_data_handler<
     D: KvStoreConnection + Clone + Send + 'static,
     C: KvStoreConnection + CacheHandler + Clone + Send + 'static,
@@ -172,3 +168,54 @@ pub async fn set_data_handler<
         Err(_) => r.into_err_internal(ApiErrorType::CuckooFilterInsertionFailed),
     }
 }
+
+/// Route to del data from DB
+///
+/// /// ### Arguments
+///
+/// * `headers` - Request headers
+/// * `value_id` - Value ID to retrieve (Optional, if not provided, all values for the address are deleted)
+/// * `db` - Database connection
+/// * `cache` - Cache connection
+/// * `c_filter` - Cuckoo filter connection
+pub async fn del_data_handler<
+    D: KvStoreConnection + Clone + Send + 'static,
+    C: KvStoreConnection + Clone + Send + 'static,
+>(
+    headers: warp::hyper::HeaderMap,
+    value_id: Option<String>,
+    db: Arc<Mutex<D>>,
+    cache: Arc<Mutex<C>>,
+    c_filter: CFilterConnection,
+) -> Result<JsonReply, JsonReply> {
+    let r = CallResponse::new("del_data");
+    info!("DEL_DATA requested with headers: {:?}", headers);
+
+    let address = headers
+        .get("address")
+        .and_then(|n| n.to_str().ok())
+        .unwrap_or_default();
+
+    // delete address in cuckoo filter if no value_id is provided
+    if value_id.is_none() && !c_filter.lock().await.delete(&address) {
+        error!("Address not found in cuckoo filter");
+        return r.into_err_internal(ApiErrorType::CuckooFilterLookupFailed);
+    }
+
+    // Check cache
+    let mut cache_lock_result = cache.lock().await;
+    let cache_result = cache_lock_result
+        .del_data(address, value_id.as_deref())
+        .await;
+
+    match cache_result {
+        Ok(_) => {
+            debug!("Data deleted from cache");
+            return delete_from_db(db, address, value_id.as_deref()).await
+        }
+        Err(_) => {
+            error!("Cache deletion failed for address: {}", address);
+            return r.into_err_internal(ApiErrorType::CacheDeleteFailed);
+        }
+    }
+}

src/api/routes.rs

@@ -1,4 +1,4 @@
-use crate::api::handlers::{get_data_handler, set_data_handler};
+use crate::api::handlers::{get_data_handler, set_data_handler, del_data_handler};
 use crate::db::handler::{CacheHandler, KvStoreConnection};
 use futures::lock::Mutex;
 use std::sync::Arc;
@@ -40,7 +40,7 @@ pub fn get_data_with_id<
         .and(with_node_component(cuckoo_filter))
         .and_then(move |_, headers, value_id: String, cache, db, cf| {
             // Add type annotation for headers parameter
-            debug!("GET_DATA requested");
+            debug!("GET_DATA requested with value_id({:?})", value_id);
             map_api_res(get_data_handler(headers, Some(value_id), db, cache, cf))
         })
         .with(get_cors())
@@ -108,3 +108,37 @@ pub fn set_data<
         })
         .with(post_cors())
 }
+
+/// DELETE /del_data
+///
+/// Deletes all data associated with a given address
+///
+/// ### Arguments
+///
+/// * `db` - The database connection to use
+/// * `cache` - The cache connection to use
+/// * `cuckoo_filter` - The cuckoo filter connection to use
+pub fn del_data<
+    D: KvStoreConnection + Clone + Send + 'static,
+    C: KvStoreConnection + Clone + Send + 'static,
+>(
+    db: Arc<Mutex<D>>,
+    cache: Arc<Mutex<C>>,
+    cuckoo_filter: CFilterConnection,
+) -> impl Filter<Extract = (impl Reply,), Error = Rejection> + Clone {
+    debug!("Setting up del_data route");
+
+    warp::path("del_data")
+        .and(warp::delete())
+        .and(sig_verify_middleware())
+        .and(warp::header::headers_cloned())
+        .and(with_node_component(cache))
+        .and(with_node_component(db))
+        .and(with_node_component(cuckoo_filter))
+        .and_then(move |_, headers, cache, db, cf| {
+            // Add type annotation for headers parameter
+            debug!("DEL_DATA requested");
+            map_api_res(del_data_handler(headers, None, db, cache, cf))
+        })
+        .with(get_cors())
+}

src/api/utils.rs

@@ -13,6 +13,7 @@ use valence_core::api::responses::{json_serialize_embed, CallResponse, JsonReply
 ///
 /// * `db` - Database connection
 /// * `address` - Address to retrieve data from
+/// * `value_id` - Value ID to retrieve (Optional, if not provided, all values for the address are retrieved)
 pub async fn retrieve_from_db<D: KvStoreConnection + Clone + Send + 'static>(
     db: Arc<Mutex<D>>,
     address: &str,
@@ -21,32 +22,54 @@ pub async fn retrieve_from_db<D: KvStoreConnection + Clone + Send + 'static>(
     let r = CallResponse::new("get_data");
     info!("RETRIEVE_FROM_DB requested with address: {:?}", address);
 
-    let db_result: Result<Option<HashMap<String, String>>, _> =
-        db.lock().await.get_data(&address, value_id).await;
-    let value_id = value_id.unwrap().to_string();
+    let db_result: Result<Option<HashMap<String, Value>>, _> =
+        db.lock().await.get_data(address, value_id).await;
 
     match db_result {
         Ok(data) => match data {
             Some(value) => {
-                info!("Data retrieved from DB");
-                let data = value
-                    .get(&value_id)
-                    .iter()
-                    .map(|v| serde_json::from_str(v).unwrap())
-                    .collect::<Vec<Value>>();
-                return r.into_ok("Data retrieved successfully", json_serialize_embed(data));
+                return r.into_ok("Data retrieved successfully", json_serialize_embed(value));
             }
             None => {
-                info!("Data not found in DB");
-                return r.into_err_internal(ApiErrorType::Generic("Data not found".to_string()));
+                return r.into_err_internal(ApiErrorType::DataNotFound);
             }
         },
-        Err(_) => r.into_err_internal(ApiErrorType::Generic(
-            "Full Valence chain retrieval failed".to_string(),
-        )),
+        Err(_) => r.into_err_internal(ApiErrorType::DBQueryFailed),
     }
 }
 
+/// Deletes data from the database
+///
+/// ### Arguments
+///
+/// * `db` - Database connection
+/// * `address` - Address to retrieve data from
+/// * `value_id` - Value ID to delete (Optional, if not provided, all values for the address are deleted)
+pub async fn delete_from_db<D: KvStoreConnection + Clone + Send + 'static>(
+    db: Arc<Mutex<D>>,
+    address: &str,
+    value_id: Option<&str>,
+) -> Result<JsonReply, JsonReply> {
+    let r = CallResponse::new("del_data");
+    info!("DELETE_FROM_DB requested with address: {:?}", address);
+
+    let db_result = db.lock().await.del_data(address, value_id).await;
+
+    match db_result {
+        Ok(_) => r.into_ok("Data deleted successfully", json_serialize_embed(address)),
+        Err(_) => r.into_err_internal(ApiErrorType::Generic(format!(
+            "{:?} for {:?}",
+            ApiErrorType::ValueDeleteFailed,
+            address
+        ))),
+    }
+}
+
+/// Serialize all entries in a HashMap
+///
+/// ### Arguments
+///
+/// * `data` - HashMap of key-value pairs to serialize
 pub fn serialize_all_entries(data: HashMap<String, String>) -> HashMap<String, Value> {
     let mut output = HashMap::new();
 
@@ -60,6 +83,5 @@ pub fn serialize_all_entries(data: HashMap<String, String>) -> HashMap<String, V
             }
         }
     }
-
     output
 }