benchmarks

Cargo.toml

@@ -17,7 +17,8 @@ members = [
     "packages/plugin",
     "packages/native",
     "packages/resolver-extensions",
-    "packages/default-config"
+    "packages/default-config",
+    "packages/benchmarks"
 ]
 
 # Do not include test utils & default config crates in default members
@@ -35,7 +36,7 @@ default-members = [
 ]
 
 [workspace.dependencies]
-polywrap_client = { version = "0.1.6-beta.7", path = "packages/client" } 
+polywrap_client = { version = "0.1.6-beta.7", path = "packages/client" }
 polywrap_wasm = { version = "0.1.6-beta.7", path = "packages/wasm" }
 polywrap_client_builder = { version = "0.1.6-beta.7", path = "packages/builder" }
 polywrap_core = { version = "0.1.6-beta.7", path = "packages/core" }

packages/benchmarks/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "polywrap_benchmarks"
+description = "Polywrap Client benchmarks in rust"
+repository = "https://github.com/polywrap/rust-client"
+
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+
+[lib]
+crate-type = ["rlib"]
+
+[dependencies]
+polywrap_client.workspace = true
+polywrap_wasm.workspace = true
+polywrap_client_builder.workspace = true
+polywrap_core.workspace = true
+wrap_manifest_schemas.workspace = true
+polywrap_resolvers.workspace = true
+polywrap_msgpack.workspace = true
+polywrap_tests_utils.workspace = true
+polywrap_plugin.workspace = true
+polywrap_plugin_implementor.workspace = true
+polywrap_client_default_config.workspace = true
+polywrap-wasm-rs = "*"
+
+[dev-dependencies]
+criterion = { version = "0.5.1", features = ["html_reports"] }
+
+[[bench]]
+name = "bench_invoke"
+harness = false
+
+[[bench]]
+name = "bench_runtime"
+harness = false
+
+[[bench]]
+name = "bench_load_wrapper"
+harness = false

packages/benchmarks/benches/bench_invoke.rs

@@ -0,0 +1,97 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use polywrap_client::client::PolywrapClient;
+use polywrap_client::core::uri::Uri;
+use polywrap_core::client::ClientConfig;
+use polywrap_core::file_reader::SimpleFileReader;
+use polywrap_core::resolution::uri_resolution_context::UriPackageOrWrapper;
+use polywrap_msgpack::msgpack;
+use polywrap_resolvers::base_resolver::BaseResolver;
+use polywrap_resolvers::simple_file_resolver::FilesystemResolver;
+use polywrap_resolvers::static_resolver::StaticResolver;
+use polywrap_tests_utils::helpers::get_tests_path_string;
+
+fn prepare_client() -> PolywrapClient {
+    let path = get_tests_path_string();
+    let subinvoke_uri = Uri::try_from(format!(
+        "fs/{path}/subinvoke/00-subinvoke/implementations/rs"
+    )).unwrap();
+
+    let mut resolvers = HashMap::new();
+    resolvers.insert(
+        String::from("wrap://ens/imported-subinvoke.eth"),
+        UriPackageOrWrapper::Uri(subinvoke_uri),
+    );
+    let file_reader = SimpleFileReader::new();
+    let fs_resolver = FilesystemResolver::new(Arc::new(file_reader));
+
+    let base_resolver = BaseResolver::new(
+        Box::new(fs_resolver),
+        Box::new(StaticResolver::new(resolvers)),
+    );
+
+    let config = ClientConfig {
+        resolver: Arc::new(base_resolver),
+        envs: None,
+        interfaces: None,
+    };
+    let client = PolywrapClient::new(config);
+    client
+}
+
+fn bench_invoke(c: &mut Criterion) {
+    let client = prepare_client();
+
+    // Note: this is using the correct URI for invoke, which is in the 00-subinvoke wrapper
+    let path = get_tests_path_string();
+    let uri = Uri::try_from(format!(
+        "fs/{path}/subinvoke/00-subinvoke/implementations/rs"
+    )).unwrap();
+
+    c.bench_function("client/invoke", |b| b.iter(|| {
+        let result = client
+            .invoke::<u32>(
+                &uri,
+                "add",
+                Some(&msgpack!({"a": 1, "b": 1})),
+                None,
+                None,
+            )
+            .unwrap();
+
+        assert_eq!(result, 2);
+    }));
+}
+
+fn bench_subinvoke(c: &mut Criterion) {
+    let client = prepare_client();
+
+    // Note: this is using the correct URI for subinvoke, which is in the 01-invoke wrapper
+    let path = get_tests_path_string();
+    let uri = Uri::try_from(format!(
+        "fs/{path}/subinvoke/01-invoke/implementations/rs"
+    )).unwrap();
+
+    c.bench_function("client/subinvoke", |b| b.iter(|| {
+        let result = client
+            .invoke::<u32>(
+                &uri,
+                "addAndIncrement",
+                Some(&msgpack!({"a": 1, "b": 1})),
+                None,
+                None,
+            )
+            .unwrap();
+
+        assert_eq!(result, 3);
+    }));
+}
+
+criterion_group!{
+    name = benches;
+    config = Criterion::default().sample_size(60);
+    targets = bench_invoke, bench_subinvoke
+}
+criterion_main!(benches);

packages/benchmarks/benches/bench_load_wrapper.rs

@@ -0,0 +1,79 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use criterion::BatchSize::PerIteration;
+
+use polywrap_client::client::PolywrapClient;
+use polywrap_core::{
+    uri::Uri,
+    wrap_loader::WrapLoader,
+};
+use polywrap_client_builder::types::{ClientConfigHandler};
+use polywrap_core::uri_resolver_handler::UriResolverHandler;
+use polywrap_benchmarks::{get_fibonacci_dir};
+
+fn prepare_client() -> PolywrapClient {
+    let builder = polywrap_client_default_config::build();
+    let config = builder.build();
+    PolywrapClient::new(config)
+}
+
+pub struct UriCase {
+    pub id: String,
+    pub uri: Uri,
+}
+
+pub fn prepare_uris() -> Vec<UriCase> {
+    let fs_uri = UriCase {
+        id: "fs_uri".to_string(),
+        uri: Uri::try_from(format!("fs/{}", get_fibonacci_dir("rs"))).unwrap(),
+    };
+    let http_uri = UriCase {
+        id: "http_uri".to_string(),
+        uri: Uri::try_from(format!(
+            "http/https://raw.githubusercontent.com/polywrap/rust-client/kris/debug-slow-invocation/packages/benchmarks/fibonacci/rs/build"
+        )).unwrap(),
+    };
+    let ipfs_uri = UriCase {
+        id: "ipfs_uri".to_string(),
+        uri: Uri::try_from("ipfs/QmXTYY4HAhurxZURrnRM8uD2oBCog8ATYDruVekuXQB192").unwrap(),
+    };
+    vec![fs_uri, http_uri, ipfs_uri]
+}
+
+fn bench_try_resolve_uri(c: &mut Criterion) {
+    let uri_list = prepare_uris();
+
+    let mut group = c.benchmark_group("client/try_resolve_uri");
+
+    for uri in uri_list {
+        group.bench_with_input(&uri.id, &uri.uri, |b, uri| b.iter_batched_ref(|| {
+            prepare_client()
+        }, | client: &mut PolywrapClient | {
+            client.try_resolve_uri(uri, None).unwrap();
+        },
+      PerIteration
+        ));
+    }
+}
+
+fn bench_load_wrapper(c: &mut Criterion) {
+    let uri_list = prepare_uris();
+
+    let mut group = c.benchmark_group("client/load_wrapper");
+
+    for uri in uri_list {
+        group.bench_with_input(&uri.id, &uri.uri, |b, uri| b.iter_batched_ref(|| {
+            prepare_client()
+        }, | client: &mut PolywrapClient | {
+            client.load_wrapper(uri, None).unwrap();
+        },
+        PerIteration
+        ));
+    }
+}
+
+criterion_group!{
+    name = benches;
+    config = Criterion::default().sample_size(10);
+    targets = bench_try_resolve_uri, bench_load_wrapper
+}
+criterion_main!(benches);

packages/benchmarks/benches/bench_runtime.rs

@@ -0,0 +1,117 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::{path::Path};
+use polywrap_wasm::{wasm_wrapper::{WasmWrapper}};
+use polywrap_core::{
+    file_reader::{SimpleFileReader},
+    wrapper::Wrapper,
+};
+
+use polywrap_msgpack::msgpack;
+use std::sync::{Arc};
+use std::fs;
+use polywrap_tests_utils::mocks::MockInvoker;
+use polywrap_benchmarks::{get_fibonacci_wrap};
+use polywrap_benchmarks::fibonacci::{fibonacci_loop, fibonacci_recursive};
+use polywrap_tests_utils::helpers::get_tests_path_string;
+
+fn bench_invoke(c: &mut Criterion) {
+    // Note: this is using the correct URI for invoke, which is in the 00-subinvoke wrapper
+    let path = get_tests_path_string();
+    let module_path = format!("{path}/subinvoke/00-subinvoke/implementations/as/wrap.wasm");
+
+    let module_bytes = fs::read(Path::new(&module_path)).unwrap();
+    let file_reader = SimpleFileReader::new();
+    let wrapper = WasmWrapper::new(module_bytes, Arc::new(file_reader));
+
+    let mock_invoker = Arc::new(MockInvoker {});
+
+    c.bench_function("wasm/invoke", |b| b.iter(|| {
+        let result = wrapper.invoke(
+            "add",
+            Some(&msgpack!({ "a": 1, "b": 1})),
+            None,
+            mock_invoker.clone(),
+            None
+        ).unwrap();
+        assert_eq!(result, [2]);
+    }));
+}
+
+fn bench_fibonacci_loop(c: &mut Criterion) {
+    let mock_invoker = Arc::new(MockInvoker {});
+    let n = 10_000;
+
+    let mut group = c.benchmark_group("wasm/fibonacci_loop");
+
+    group.bench_function("native", |b| b.iter(|| {
+        fibonacci_loop(n).unwrap()
+    }));
+
+    let wrapper = get_fibonacci_wrap("as");
+    group.bench_function("as", |b| b.iter(|| {
+        wrapper.invoke(
+            "fibonacci_loop",
+            Some(&msgpack!({ "n": n })),
+            None,
+            mock_invoker.clone(),
+            None
+        ).unwrap();
+    }));
+
+    let wrapper = get_fibonacci_wrap("rs");
+    group.bench_function("rs", |b| b.iter(|| {
+        wrapper.invoke(
+            "fibonacci_loop",
+            Some(&msgpack!({ "n": n })),
+            None,
+            mock_invoker.clone(),
+            None
+        ).unwrap();
+    }));
+
+    group.finish();
+}
+
+fn bench_fibonacci_recursive(c: &mut Criterion) {
+    let mock_invoker = Arc::new(MockInvoker {});
+
+    // AS wrap crashes at higher N
+    let n = 30;
+
+    let mut group = c.benchmark_group("wasm/fibonacci_recursive");
+
+    group.bench_function("native", |b| b.iter(|| {
+        fibonacci_recursive(n).unwrap()
+    }));
+
+    let wrapper = get_fibonacci_wrap("as");
+    group.bench_function("as", |b| b.iter(|| {
+        wrapper.invoke(
+            "fibonacci_recursive",
+            Some(&msgpack!({ "n": n })),
+            None,
+            mock_invoker.clone(),
+            None
+        ).unwrap();
+    }));
+
+    let wrapper = get_fibonacci_wrap("rs");
+    group.bench_function("rs", |b| b.iter(|| {
+        wrapper.invoke(
+            "fibonacci_recursive",
+            Some(&msgpack!({ "n": n })),
+            None,
+            mock_invoker.clone(),
+            None
+        ).unwrap();
+    }));
+
+    group.finish();
+}
+
+criterion_group!{
+    name = benches;
+    config = Criterion::default().sample_size(10);
+    targets = bench_invoke, bench_fibonacci_loop, bench_fibonacci_recursive
+}
+criterion_main!(benches);

packages/benchmarks/fibonacci/as/.gitignore

@@ -0,0 +1,5 @@
+wrap
+node_modules
+.polywrap
+yarn.lock
+Cargo.lock

packages/benchmarks/fibonacci/as/URI.txt

@@ -0,0 +1 @@
+wrap://ipfs/QmYjsgLJ9z9Z5BytJip1JYC3yQ5mSoy1svd6afzoK3swH1

packages/benchmarks/fibonacci/as/build/wrap.info

@@ -0,0 +1 @@
+��version�0.1�name�fibonacci-as�type�wasm�abi��version�0.1�moduleType��type�Module�kind̀�methods���name�fibonacci_loop�return��type�BigInt�name�fibonacci_loop�requiredäkind"�scalar��name�fibonacci_loop�type�BigInt�requiredäkind�type�Method�kind@�requiredéarguments���type�Int�name�n�requiredäkind"�scalar��name�n�type�Int�requiredäkind��name�fibonacci_recursive�return��type�BigInt�name�fibonacci_recursive�requiredäkind"�scalar��name�fibonacci_recursive�type�BigInt�requiredäkind�type�Method�kind@�requiredéarguments���type�Int�name�n�requiredäkind"�scalar��name�n�type�Int�requiredäkind

packages/benchmarks/fibonacci/as/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "test-cases-default",
+  "private": true,
+  "scripts": {
+    "build": "npx polywrap build",
+    "codegen": "npx polywrap codegen"
+  },
+  "dependencies": {
+    "@polywrap/wasm-as": "latest",
+    "assemblyscript": "0.19.23"
+  },
+  "devDependencies": {
+    "polywrap": "latest"
+  }
+}

packages/benchmarks/fibonacci/as/polywrap.deploy.yaml

@@ -0,0 +1,9 @@
+format: 0.2.0
+jobs:
+  deploy:
+    steps:
+      - name: ipfs_deploy
+        package: ipfs
+        uri: fs/./build
+        config:
+          gatewayUri: https://ipfs.wrappers.io