From 871e29d92b222267ac4bc2849a81592789a103ad Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Mon, 3 Mar 2025 10:52:00 +0000 Subject: [PATCH 1/3] ObjectStore WASM32 Support --- .github/workflows/object_store.yml | 2 +- object_store/src/aws/builder.rs | 6 ++--- object_store/src/aws/mod.rs | 4 ++++ object_store/src/azure/builder.rs | 6 ++--- object_store/src/client/body.rs | 1 + object_store/src/client/connection.rs | 34 ++++++++++++++++++++++++++- object_store/src/client/mod.rs | 22 ++++++++++++----- object_store/src/gcp/builder.rs | 6 ++--- object_store/src/http/mod.rs | 7 ++---- object_store/src/lib.rs | 17 ++++---------- object_store/src/parse.rs | 8 ++++++- 11 files changed, 75 insertions(+), 38 deletions(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index 1639b031ebfc..d9e4fcbd1363 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -208,7 +208,7 @@ jobs: - name: Build wasm32-unknown-unknown run: cargo build --target wasm32-unknown-unknown - name: Build wasm32-wasip1 - run: cargo build --target wasm32-wasip1 + run: cargo build --all-features --target wasm32-wasip1 windows: name: cargo test LocalFileSystem (win64) diff --git a/object_store/src/aws/builder.rs b/object_store/src/aws/builder.rs index e49ba1dcfe48..3c2d2d022d30 100644 --- a/object_store/src/aws/builder.rs +++ b/object_store/src/aws/builder.rs @@ -23,7 +23,7 @@ use crate::aws::{ AmazonS3, AwsCredential, AwsCredentialProvider, Checksum, S3ConditionalPut, S3CopyIfNotExists, STORE, }; -use crate::client::{HttpConnector, ReqwestConnector, TokenCredentialProvider}; +use crate::client::{http_connector, HttpConnector, TokenCredentialProvider}; use crate::config::ConfigValue; use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider}; use base64::prelude::BASE64_STANDARD; @@ -896,9 +896,7 @@ impl AmazonS3Builder { self.parse_url(&url)?; } - let http = self - .http_connector - .unwrap_or_else(|| Arc::new(ReqwestConnector::default())); + let http = http_connector(self.http_connector)?; let bucket = self.bucket_name.ok_or(Error::MissingBucketName)?; let region = self.region.unwrap_or_else(|| "us-east-1".to_string()); diff --git a/object_store/src/aws/mod.rs b/object_store/src/aws/mod.rs index 76e298fe1bbb..b8175bd1e6f0 100644 --- a/object_store/src/aws/mod.rs +++ b/object_store/src/aws/mod.rs @@ -58,12 +58,16 @@ mod client; mod credential; mod dynamo; mod precondition; + +#[cfg(not(target_arch = "wasm32"))] mod resolve; pub use builder::{AmazonS3Builder, AmazonS3ConfigKey}; pub use checksum::Checksum; pub use dynamo::DynamoCommit; pub use precondition::{S3ConditionalPut, S3CopyIfNotExists}; + +#[cfg(not(target_arch = "wasm32"))] pub use resolve::resolve_bucket_region; /// This struct is used to maintain the URI path encoding diff --git a/object_store/src/azure/builder.rs b/object_store/src/azure/builder.rs index ab0a484f5463..83dcf0754c1d 100644 --- a/object_store/src/azure/builder.rs +++ b/object_store/src/azure/builder.rs @@ -21,7 +21,7 @@ use crate::azure::credential::{ ImdsManagedIdentityProvider, WorkloadIdentityOAuthProvider, }; use crate::azure::{AzureCredential, AzureCredentialProvider, MicrosoftAzure, STORE}; -use crate::client::{HttpConnector, ReqwestConnector, TokenCredentialProvider}; +use crate::client::{http_connector, HttpConnector, TokenCredentialProvider}; use crate::config::ConfigValue; use crate::{ClientConfigKey, ClientOptions, Result, RetryConfig, StaticCredentialProvider}; use percent_encoding::percent_decode_str; @@ -907,9 +907,7 @@ impl MicrosoftAzureBuilder { Arc::new(StaticCredentialProvider::new(credential)) }; - let http = self - .http_connector - .unwrap_or_else(|| Arc::new(ReqwestConnector::default())); + let http = http_connector(self.http_connector)?; let (is_emulator, storage_url, auth, account) = if self.use_emulator.get()? { let account_name = self diff --git a/object_store/src/client/body.rs b/object_store/src/client/body.rs index 549b3e4daf20..8f62afa4ff2e 100644 --- a/object_store/src/client/body.rs +++ b/object_store/src/client/body.rs @@ -39,6 +39,7 @@ impl HttpRequestBody { Self(Inner::Bytes(Bytes::new())) } + #[cfg(not(target_arch = "wasm32"))] pub(crate) fn into_reqwest(self) -> reqwest::Body { match self.0 { Inner::Bytes(b) => b.into(), diff --git a/object_store/src/client/connection.rs b/object_store/src/client/connection.rs index 8b631694a57b..7e2daf4cdb81 100644 --- a/object_store/src/client/connection.rs +++ b/object_store/src/client/connection.rs @@ -84,9 +84,14 @@ impl HttpError { } pub(crate) fn reqwest(e: reqwest::Error) -> Self { + #[cfg(not(target_arch = "wasm32"))] + let is_connect = || e.is_connect(); + #[cfg(target_arch = "wasm32")] + let is_connect = || false; + let mut kind = if e.is_timeout() { HttpErrorKind::Timeout - } else if e.is_connect() { + } else if is_connect() { HttpErrorKind::Connect } else if e.is_decode() { HttpErrorKind::Decode @@ -200,6 +205,7 @@ impl HttpClient { } #[async_trait] +#[cfg(not(target_arch = "wasm32"))] impl HttpService for reqwest::Client { async fn call(&self, req: HttpRequest) -> Result { let (parts, body) = req.into_parts(); @@ -227,11 +233,37 @@ pub trait HttpConnector: std::fmt::Debug + Send + Sync + 'static { /// [`HttpConnector`] using [`reqwest::Client`] #[derive(Debug, Default)] #[allow(missing_copy_implementations)] +#[cfg(not(target_arch = "wasm32"))] pub struct ReqwestConnector {} +#[cfg(not(target_arch = "wasm32"))] impl HttpConnector for ReqwestConnector { fn connect(&self, options: &ClientOptions) -> crate::Result { let client = options.client()?; Ok(HttpClient::new(client)) } } + +#[cfg(target_arch = "wasm32")] +pub(crate) fn http_connector( + custom: Option>, +) -> crate::Result> { + match custom { + Some(x) => Ok(x), + None => Err(crate::Error::NotSupported { + source: "WASM32 architectures must provide an HTTPConnector" + .to_string() + .into(), + }), + } +} + +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn http_connector( + custom: Option>, +) -> crate::Result> { + match custom { + Some(x) => Ok(x), + None => Ok(Arc::new(ReqwestConnector {})), + } +} diff --git a/object_store/src/client/mod.rs b/object_store/src/client/mod.rs index 36252f54f18c..07b576c7a6c4 100644 --- a/object_store/src/client/mod.rs +++ b/object_store/src/client/mod.rs @@ -19,6 +19,7 @@ pub(crate) mod backoff; +#[cfg(not(target_arch = "wasm32"))] mod dns; #[cfg(test)] @@ -48,22 +49,25 @@ pub use body::{HttpRequest, HttpRequestBody, HttpResponse, HttpResponseBody}; pub(crate) mod builder; mod connection; -pub use connection::{ - HttpClient, HttpConnector, HttpError, HttpErrorKind, HttpService, ReqwestConnector, -}; +pub(crate) use connection::http_connector; +#[cfg(not(target_arch = "wasm32"))] +pub use connection::ReqwestConnector; +pub use connection::{HttpClient, HttpConnector, HttpError, HttpErrorKind, HttpService}; #[cfg(any(feature = "aws", feature = "gcp", feature = "azure"))] pub(crate) mod parts; use async_trait::async_trait; use reqwest::header::{HeaderMap, HeaderValue}; -use reqwest::{Client, ClientBuilder, NoProxy, Proxy}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; +#[cfg(not(target_arch = "wasm32"))] +use reqwest::{NoProxy, Proxy}; + use crate::config::{fmt_duration, ConfigValue}; use crate::path::Path; use crate::{GetOptions, Result}; @@ -195,8 +199,10 @@ impl FromStr for ClientConfigKey { /// This is used to configure the client to trust a specific certificate. See /// [Self::from_pem] for an example #[derive(Debug, Clone)] +#[cfg(not(target_arch = "wasm32"))] pub struct Certificate(reqwest::tls::Certificate); +#[cfg(not(target_arch = "wasm32"))] impl Certificate { /// Create a `Certificate` from a PEM encoded certificate. /// @@ -243,6 +249,7 @@ impl Certificate { #[derive(Debug, Clone)] pub struct ClientOptions { user_agent: Option>, + #[cfg(not(target_arch = "wasm32"))] root_certificates: Vec, content_type_map: HashMap, default_content_type: Option, @@ -276,6 +283,7 @@ impl Default for ClientOptions { // we opt for a slightly higher default timeout of 30 seconds Self { user_agent: None, + #[cfg(not(target_arch = "wasm32"))] root_certificates: Default::default(), content_type_map: Default::default(), default_content_type: None, @@ -402,6 +410,7 @@ impl ClientOptions { /// /// This can be used to connect to a server that has a self-signed /// certificate for example. + #[cfg(not(target_arch = "wasm32"))] pub fn with_root_certificate(mut self, certificate: Certificate) -> Self { self.root_certificates.push(certificate); self @@ -614,8 +623,9 @@ impl ClientOptions { .with_connect_timeout(Duration::from_secs(1)) } - pub(crate) fn client(&self) -> Result { - let mut builder = ClientBuilder::new(); + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn client(&self) -> Result { + let mut builder = reqwest::ClientBuilder::new(); match &self.user_agent { Some(user_agent) => builder = builder.user_agent(user_agent.get()?), diff --git a/object_store/src/gcp/builder.rs b/object_store/src/gcp/builder.rs index 793978355767..48b3637f9fb8 100644 --- a/object_store/src/gcp/builder.rs +++ b/object_store/src/gcp/builder.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::client::{HttpConnector, ReqwestConnector, TokenCredentialProvider}; +use crate::client::{http_connector, HttpConnector, TokenCredentialProvider}; use crate::gcp::client::{GoogleCloudStorageClient, GoogleCloudStorageConfig}; use crate::gcp::credential::{ ApplicationDefaultCredentials, InstanceCredentialProvider, ServiceAccountCredentials, @@ -442,9 +442,7 @@ impl GoogleCloudStorageBuilder { let bucket_name = self.bucket_name.ok_or(Error::MissingBucketName {})?; - let http = self - .http_connector - .unwrap_or_else(|| Arc::new(ReqwestConnector::default())); + let http = http_connector(self.http_connector)?; // First try to initialize from the service account information. let service_account_credentials = diff --git a/object_store/src/http/mod.rs b/object_store/src/http/mod.rs index 8fba4d7c67c7..34489d76de3f 100644 --- a/object_store/src/http/mod.rs +++ b/object_store/src/http/mod.rs @@ -41,7 +41,7 @@ use url::Url; use crate::client::get::GetClientExt; use crate::client::header::get_etag; -use crate::client::{HttpConnector, ReqwestConnector}; +use crate::client::{http_connector, HttpConnector}; use crate::http::client::Client; use crate::path::Path; use crate::{ @@ -248,10 +248,7 @@ impl HttpBuilder { let url = self.url.ok_or(Error::MissingUrl)?; let parsed = Url::parse(&url).map_err(|source| Error::UnableToParseUrl { url, source })?; - let client = match self.http_connector { - None => ReqwestConnector::default().connect(&self.client_options)?, - Some(x) => x.connect(&self.client_options)?, - }; + let client = http_connector(self.http_connector)?.connect(&self.client_options)?; Ok(HttpStore { client: Arc::new(Client::new( diff --git a/object_store/src/lib.rs b/object_store/src/lib.rs index 5db7e01d7d89..836cd75bc59b 100644 --- a/object_store/src/lib.rs +++ b/object_store/src/lib.rs @@ -497,12 +497,6 @@ //! [`webpki-roots`]: https://crates.io/crates/webpki-roots //! -#[cfg(all( - target_arch = "wasm32", - any(feature = "gcp", feature = "aws", feature = "azure", feature = "http") -))] -compile_error!("Features 'gcp', 'aws', 'azure', 'http' are not supported on wasm."); - #[cfg(feature = "aws")] pub mod aws; #[cfg(feature = "azure")] @@ -530,10 +524,13 @@ pub mod client; #[cfg(feature = "cloud")] pub use client::{ - backoff::BackoffConfig, retry::RetryConfig, Certificate, ClientConfigKey, ClientOptions, - CredentialProvider, StaticCredentialProvider, + backoff::BackoffConfig, retry::RetryConfig, ClientConfigKey, ClientOptions, CredentialProvider, + StaticCredentialProvider, }; +#[cfg(all(feature = "cloud", not(target_arch = "wasm32")))] +pub use client::Certificate; + #[cfg(feature = "cloud")] mod config; @@ -1083,8 +1080,6 @@ impl GetResult { .await } GetResultPayload::Stream(s) => collect_bytes(s, Some(len)).await, - #[cfg(target_arch = "wasm32")] - _ => unimplemented!("File IO not implemented on wasm32."), } } @@ -1110,8 +1105,6 @@ impl GetResult { local::chunked_stream(file, path, self.range, CHUNK_SIZE) } GetResultPayload::Stream(s) => s, - #[cfg(target_arch = "wasm32")] - _ => unimplemented!("File IO not implemented on wasm32."), } } } diff --git a/object_store/src/parse.rs b/object_store/src/parse.rs index 4e67e5946a82..00ea6cf23908 100644 --- a/object_store/src/parse.rs +++ b/object_store/src/parse.rs @@ -201,7 +201,13 @@ where let url = &url[..url::Position::BeforePath]; builder_opts!(crate::http::HttpBuilder, url, _options) } - #[cfg(not(all(feature = "aws", feature = "azure", feature = "gcp", feature = "http")))] + #[cfg(not(all( + feature = "aws", + feature = "azure", + feature = "gcp", + feature = "http", + not(target_arch = "wasm32") + )))] s => { return Err(super::Error::Generic { store: "parse_url", From 569e63d82e4c61c7af8c7b7e7de06a8433dde402 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Mon, 3 Mar 2025 11:03:30 +0000 Subject: [PATCH 2/3] Docs --- .github/workflows/object_store.yml | 2 ++ object_store/src/aws/builder.rs | 4 +++- object_store/src/azure/builder.rs | 4 +++- object_store/src/client/mod.rs | 2 +- object_store/src/gcp/builder.rs | 4 +++- object_store/src/http/mod.rs | 4 +++- 6 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index d9e4fcbd1363..9952c32375c7 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -205,6 +205,8 @@ jobs: uses: ./.github/actions/setup-builder with: target: wasm32-unknown-unknown,wasm32-wasip1 + - name: Install clag + run: apt-get update && apt-get install -y clang - name: Build wasm32-unknown-unknown run: cargo build --target wasm32-unknown-unknown - name: Build wasm32-wasip1 diff --git a/object_store/src/aws/builder.rs b/object_store/src/aws/builder.rs index 3c2d2d022d30..5dff94d0c3b7 100644 --- a/object_store/src/aws/builder.rs +++ b/object_store/src/aws/builder.rs @@ -883,7 +883,9 @@ impl AmazonS3Builder { self } - /// Overrides the [`HttpConnector`], by default uses [`ReqwestConnector`] + /// The [`HttpConnector`] to use + /// + /// On non-WASM32 platforms uses [`reqwest`] by default, on WASM32 platforms must be provided pub fn with_http_connector(mut self, connector: C) -> Self { self.http_connector = Some(Arc::new(connector)); self diff --git a/object_store/src/azure/builder.rs b/object_store/src/azure/builder.rs index 83dcf0754c1d..f176fc680ff4 100644 --- a/object_store/src/azure/builder.rs +++ b/object_store/src/azure/builder.rs @@ -889,7 +889,9 @@ impl MicrosoftAzureBuilder { self } - /// Overrides the [`HttpConnector`], by default uses [`ReqwestConnector`] + /// The [`HttpConnector`] to use + /// + /// On non-WASM32 platforms uses [`reqwest`] by default, on WASM32 platforms must be provided pub fn with_http_connector(mut self, connector: C) -> Self { self.http_connector = Some(Arc::new(connector)); self diff --git a/object_store/src/client/mod.rs b/object_store/src/client/mod.rs index 07b576c7a6c4..bd0347b4311c 100644 --- a/object_store/src/client/mod.rs +++ b/object_store/src/client/mod.rs @@ -809,7 +809,7 @@ mod cloud { use crate::client::token::{TemporaryToken, TokenCache}; use crate::RetryConfig; - /// A [`CredentialProvider`] that uses [`Client`] to fetch temporary tokens + /// A [`CredentialProvider`] that uses [`HttpClient`] to fetch temporary tokens #[derive(Debug)] pub(crate) struct TokenCredentialProvider { inner: T, diff --git a/object_store/src/gcp/builder.rs b/object_store/src/gcp/builder.rs index 48b3637f9fb8..74aecaee2ba7 100644 --- a/object_store/src/gcp/builder.rs +++ b/object_store/src/gcp/builder.rs @@ -427,7 +427,9 @@ impl GoogleCloudStorageBuilder { self } - /// Overrides the [`HttpConnector`], by default uses [`ReqwestConnector`] + /// The [`HttpConnector`] to use + /// + /// On non-WASM32 platforms uses [`reqwest`] by default, on WASM32 platforms must be provided pub fn with_http_connector(mut self, connector: C) -> Self { self.http_connector = Some(Arc::new(connector)); self diff --git a/object_store/src/http/mod.rs b/object_store/src/http/mod.rs index 34489d76de3f..9786d83d4ac8 100644 --- a/object_store/src/http/mod.rs +++ b/object_store/src/http/mod.rs @@ -237,7 +237,9 @@ impl HttpBuilder { self } - /// Overrides the [`HttpConnector`], by default uses [`ReqwestConnector`] + /// The [`HttpConnector`] to use + /// + /// On non-WASM32 platforms uses [`reqwest`] by default, on WASM32 platforms must be provided pub fn with_http_connector(mut self, connector: C) -> Self { self.http_connector = Some(Arc::new(connector)); self From 149ca7be7dccd3362efc3fa01a646857fc0f06c5 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Mon, 3 Mar 2025 11:05:36 +0000 Subject: [PATCH 3/3] Update .github/workflows/object_store.yml --- .github/workflows/object_store.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/object_store.yml b/.github/workflows/object_store.yml index 9952c32375c7..4f71e36f4fb4 100644 --- a/.github/workflows/object_store.yml +++ b/.github/workflows/object_store.yml @@ -205,7 +205,7 @@ jobs: uses: ./.github/actions/setup-builder with: target: wasm32-unknown-unknown,wasm32-wasip1 - - name: Install clag + - name: Install clang (needed for ring) run: apt-get update && apt-get install -y clang - name: Build wasm32-unknown-unknown run: cargo build --target wasm32-unknown-unknown