Add import/export dexie db workflow to web client (#740)

CHANGELOG.md

@@ -11,6 +11,7 @@
 * Added wallet generation from seed & import from seed on web SDK (#710)
 * Add check for empty pay to id notes (#714).
 * [BREAKING] Refactored authentication out of the `Client` and added new separate authenticators (#718).
+* Added import/export for web client db (#740).
 * Re-exported RemoteTransactionProver in `rust-client` (#752).
 * Moved error handling to the `TransactionRequestBuilder::build()` (#750).
 * [BREAKING] Added starting block number parameter to `CheckNullifiersByPrefix` and removed nullifiers from `SyncState` (#758).

crates/rust-client/src/store/web_store/export/js_bindings.rs

@@ -0,0 +1,8 @@
+use wasm_bindgen::prelude::*;
+use wasm_bindgen_futures::{js_sys, wasm_bindgen};
+
+#[wasm_bindgen(module = "/src/store/web_store/js/export.js")]
+extern "C" {
+    #[wasm_bindgen(js_name = exportStore)]
+    pub fn idxdb_export_store() -> js_sys::Promise;
+}

crates/rust-client/src/store/web_store/export/mod.rs

@@ -0,0 +1,17 @@
+use super::WebStore;
+use crate::store::StoreError;
+
+mod js_bindings;
+use js_bindings::idxdb_export_store;
+use wasm_bindgen::JsValue;
+use wasm_bindgen_futures::JsFuture;
+
+impl WebStore {
+    pub async fn export_store(&self) -> Result<JsValue, StoreError> {
+        let promise = idxdb_export_store();
+        let js_value = JsFuture::from(promise)
+            .await
+            .map_err(|err| StoreError::DatabaseError(format!("Failed to export store: {err:?}")))?;
+        Ok(js_value)
+    }
+}

crates/rust-client/src/store/web_store/import/js_bindings.rs

@@ -0,0 +1,9 @@
+use wasm_bindgen::prelude::*;
+use wasm_bindgen_futures::{js_sys, wasm_bindgen};
+
+#[wasm_bindgen(module = "/src/store/web_store/js/import.js")]
+extern "C" {
+    #[wasm_bindgen(js_name = forceImportStore)]
+    pub fn idxdb_force_import_store(store_dump: JsValue) -> js_sys::Promise;
+
+}

crates/rust-client/src/store/web_store/import/mod.rs

@@ -0,0 +1,17 @@
+use super::WebStore;
+use crate::store::StoreError;
+
+mod js_bindings;
+use js_bindings::idxdb_force_import_store;
+use wasm_bindgen::JsValue;
+use wasm_bindgen_futures::JsFuture;
+
+impl WebStore {
+    pub async fn force_import_store(&self, store_dump: JsValue) -> Result<(), StoreError> {
+        let promise = idxdb_force_import_store(store_dump);
+        JsFuture::from(promise)
+            .await
+            .map_err(|err| StoreError::DatabaseError(format!("Failed to import store: {err:?}")))?;
+        Ok(())
+    }
+}

crates/rust-client/src/store/web_store/js/export.js

@@ -0,0 +1,45 @@
+import { db } from "./schema.js";
+
+async function recursivelyTransformForExport(obj) {
+  if (obj instanceof Blob) {
+    const blobBuffer = await obj.arrayBuffer();
+    return {
+      __type: "Blob",
+      data: uint8ArrayToBase64(new Uint8Array(blobBuffer)),
+    };
+  }
+
+  if (Array.isArray(obj)) {
+    return await Promise.all(obj.map(recursivelyTransformForExport));
+  }
+
+  if (obj && typeof obj === "object") {
+    const entries = await Promise.all(
+      Object.entries(obj).map(async ([key, value]) => [
+        key,
+        await recursivelyTransformForExport(value),
+      ])
+    );
+    return Object.fromEntries(entries);
+  }
+
+  return obj;
+}
+
+export async function exportStore() {
+  const db_json = {};
+  for (const table of db.tables) {
+    const records = await table.toArray();
+
+    db_json[table.name] = await Promise.all(
+      records.map(recursivelyTransformForExport)
+    );
+  }
+
+  const stringified = JSON.stringify(db_json);
+  return stringified;
+}
+
+function uint8ArrayToBase64(uint8Array) {
+  return btoa(String.fromCharCode(...uint8Array));
+}

crates/rust-client/src/store/web_store/js/import.js

@@ -0,0 +1,88 @@
+import { db, openDatabase } from "./schema.js";
+
+async function recursivelyTransformForImport(obj) {
+  if (obj && typeof obj === "object") {
+    if (obj.__type === "Blob") {
+      return new Blob([base64ToUint8Array(obj.data)]);
+    }
+
+    if (Array.isArray(obj)) {
+      return await Promise.all(obj.map(recursivelyTransformForImport));
+    }
+
+    const entries = await Promise.all(
+      Object.entries(obj).map(async ([key, value]) => [
+        key,
+        await recursivelyTransformForImport(value),
+      ])
+    );
+    return Object.fromEntries(entries);
+  }
+
+  return obj; // Return unchanged if it's neither Blob, Array, nor Object
+}
+
+export async function forceImportStore(jsonStr) {
+  try {
+    if (!db.isOpen) {
+      await openDatabase();
+    }
+
+    let db_json = JSON.parse(jsonStr);
+    if (typeof db_json === "string") {
+      db_json = JSON.parse(db_json);
+    }
+
+    const jsonTableNames = Object.keys(db_json);
+    const dbTableNames = db.tables.map((t) => t.name);
+
+    if (jsonTableNames.length === 0) {
+      throw new Error("No tables found in the provided JSON.");
+    }
+
+    // Wrap everything in a transaction
+    await db.transaction(
+      "rw",
+      ...dbTableNames.map((name) => db.table(name)),
+      async () => {
+        // Clear all tables in the database
+        await Promise.all(db.tables.map((t) => t.clear()));
+
+        // Import data from JSON into matching tables
+        for (const tableName of jsonTableNames) {
+          const table = db.table(tableName);
+
+          if (!dbTableNames.includes(tableName)) {
+            console.warn(
+              `Table "${tableName}" does not exist in the database schema. Skipping.`
+            );
+            continue; // Skip tables not in the Dexie schema
+          }
+
+          const records = db_json[tableName];
+
+          const transformedRecords = await Promise.all(
+            records.map(recursivelyTransformForImport)
+          );
+
+          await table.bulkPut(transformedRecords);
+        }
+      }
+    );
+
+    console.log("Store imported successfully.");
+  } catch (err) {
+    console.error("Failed to import store: ", err);
+    throw err;
+  }
+}
+
+function base64ToUint8Array(base64) {
+  const binaryString = atob(base64);
+  const len = binaryString.length;
+  const bytes = new Uint8Array(len);
+  for (let i = 0; i < len; i++) {
+    bytes[i] = binaryString.charCodeAt(i);
+  }
+  return bytes;
+}

crates/rust-client/src/store/web_store/mod.rs

@@ -36,6 +36,8 @@ compile_error!("The `idxdb` feature is only supported when targeting wasm32.");
 
 pub mod account;
 pub mod chain_data;
+pub mod export;
+pub mod import;
 pub mod note;
 pub mod sync;
 pub mod transaction;

crates/web-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@demox-labs/miden-sdk",
-  "version": "0.6.1-next.5",
+  "version": "0.6.1-next.6",
   "description": "Polygon Miden Wasm SDK",
   "collaborators": [
     "Polygon Miden",

crates/web-client/src/export.rs

@@ -69,4 +69,15 @@ impl WebClient {
             Err(JsValue::from_str("Client not initialized"))
         }
     }
+
+    /// Retrieves the entire underlying web store and returns it as a JsValue
+    ///
+    /// Meant to be used in conjunction with the force_import_store method
+    pub async fn export_store(&mut self) -> Result<JsValue, JsValue> {
+        let store = self.store.as_ref().ok_or(JsValue::from_str("Store not initialized"))?;
+        let export =
+            store.export_store().await.map_err(|err| JsValue::from_str(&format!("{err}")))?;
+
+        Ok(export)
+    }
 }

crates/web-client/src/import.rs

@@ -77,4 +77,17 @@ impl WebClient {
             Err(JsValue::from_str("Client not initialized"))
         }
     }
+
+    // Destructive operation, will fully overwrite the current web store
+    //
+    // The input to this function should be the result of a call to `export_store`
+    pub async fn force_import_store(&mut self, store_dump: JsValue) -> Result<JsValue, JsValue> {
+        let store = self.store.as_ref().ok_or(JsValue::from_str("Store not initialized"))?;
+        store
+            .force_import_store(store_dump)
+            .await
+            .map_err(|err| JsValue::from_str(&format!("{err}")))?;
+
+        Ok(JsValue::from_str("Store imported successfully"))
+    }
 }

crates/web-client/test/import_export.test.ts

@@ -0,0 +1,49 @@
+// TODO: Rename this / figure out rebasing with the other featuer which has import tests
+
+import { expect } from "chai";
+import { testingPage } from "./mocha.global.setup.mjs";
+import { clearStore, setupWalletAndFaucet } from "./webClientTestUtils";
+
+const exportDb = async () => {
+  return await testingPage.evaluate(async () => {
+    const client = window.client;
+    const db = await client.export_store();
+    const serialized = JSON.stringify(db);
+    return serialized;
+  });
+};
+
+const importDb = async (db: any) => {
+  return await testingPage.evaluate(async (_db) => {
+    const client = window.client;
+    await client.force_import_store(_db);
+  }, db);
+};
+
+const getAccount = async (accountId: string) => {
+  return await testingPage.evaluate(async (_accountId) => {
+    const client = window.client;
+    const accountId = window.AccountId.from_hex(_accountId);
+    const account = await client.get_account(accountId);
+    return {
+      accountId: account?.id().to_string(),
+      accountHash: account?.hash().to_hex(),
+    };
+  }, accountId);
+};
+
+describe("export and import the db", () => {
+  it("export db with an account, find the account when re-importing", async () => {
+    const { accountHash: initialAccountHash, accountId } =
+      await setupWalletAndFaucet();
+    const dbDump = await exportDb();
+
+    await clearStore();
+
+    await importDb(dbDump);
+
+    const { accountHash } = await getAccount(accountId);
+
+    expect(accountHash).to.equal(initialAccountHash);
+  });
+});

crates/web-client/test/webClientTestUtils.ts

@@ -342,6 +342,7 @@ export const consumeTransaction = async (
 interface SetupWalletFaucetResult {
   accountId: string;
   faucetId: string;
+  accountHash: string;
 }
 
 export const setupWalletAndFaucet =
@@ -363,6 +364,7 @@ export const setupWalletAndFaucet =
 
       return {
         accountId: account.id().to_string(),
+        accountHash: account.hash().to_hex(),
         faucetId: faucetAccount.id().to_string(),
       };
     });