fix(node): Fix retry logic in GCS and improve logging (#4897)

* Fix retry logic in GCS and improve logging

* Add missing change

crates/iota-config/src/object_storage_config.rs

@@ -180,17 +180,21 @@ impl ObjectStoreConfig {
         if let Some(account) = &self.google_service_account {
             builder = builder.with_service_account_path(account);
         }
+
+        let mut client_options = ClientOptions::new()
+            .with_timeout_disabled()
+            .with_connect_timeout_disabled()
+            .with_pool_idle_timeout(std::time::Duration::from_secs(300));
         if let Some(google_project_id) = &self.google_project_id {
             let x_project_header = HeaderName::from_static("x-goog-user-project");
             let iam_req_header = HeaderName::from_static("userproject");
 
             let mut headers = HeaderMap::new();
             headers.insert(x_project_header, HeaderValue::from_str(google_project_id)?);
             headers.insert(iam_req_header, HeaderValue::from_str(google_project_id)?);
-
-            builder =
-                builder.with_client_options(ClientOptions::new().with_default_headers(headers));
+            client_options = client_options.with_default_headers(headers);
         }
+        builder = builder.with_client_options(client_options);
 
         Ok(Arc::new(LimitStore::new(
             builder.build().context("invalid gcs config")?,

crates/iota-snapshot/src/reader.rs

@@ -431,7 +431,7 @@ impl StateSnapshotReaderV1 {
                         let sha3_digests_cloned = sha3_digests.clone();
                         async move {
                             // Downloads object file with retries
-                            let max_timeout = Duration::from_secs(30);
+                            let max_timeout = Duration::from_secs(60);
                             let mut timeout = Duration::from_secs(2);
                             timeout += timeout / 2;
                             timeout = std::cmp::min(max_timeout, timeout);

crates/iota-storage/src/object_store/http/gcs.rs

@@ -24,7 +24,7 @@ struct GoogleCloudStorageClient {
 
 impl GoogleCloudStorageClient {
     pub fn new(bucket: &str) -> Result<Self> {
-        let mut builder = ClientBuilder::new();
+        let mut builder = ClientBuilder::new().pool_idle_timeout(None);
         builder = builder.user_agent(DEFAULT_USER_AGENT);
         let client = builder.https_only(false).build()?;
         let bucket_name_encoded = percent_encode(bucket.as_bytes(), NON_ALPHANUMERIC).to_string();

crates/iota-storage/src/object_store/mod.rs

@@ -37,10 +37,17 @@ as_ref_get_ext_impl!(Box<dyn ObjectStoreGetExt>);
 impl ObjectStoreGetExt for Arc<DynObjectStore> {
     async fn get_bytes(&self, src: &Path) -> Result<Bytes> {
         self.get(src)
-            .await?
+            .await
+            .map_err(|e| anyhow!("Failed to get file {} with error: {:?}", src, e))?
             .bytes()
             .await
-            .map_err(|e| anyhow!("Failed to get file: {} with error: {}", src, e.to_string()))
+            .map_err(|e| {
+                anyhow!(
+                    "Failed to collect GET result for file {} into bytes with error: {:?}",
+                    src,
+                    e
+                )
+            })
     }
 }