feat(storage): Configurable RequestInit from FetchStore constructor (#77)

Author: manzt

.changeset/shy-pigs-serve.md

@@ -0,0 +1,5 @@
+---
+"@zarrita/storage": patch
+---
+
+feat: allow `RequestInit` options to be configured in `FetchStore` constructor

package.json

@@ -2,7 +2,7 @@
 	"license": "MIT",
 	"scripts": {
 		"build": "tsc --build",
-		"test": "vitest",
+		"test": "vitest --api",
 		"fmt": "deno fmt --use-tabs packages docs",
 		"lint": "deno fmt --use-tabs packages docs --check"
 	},

packages/storage/__tests__/fetch.test.ts

@@ -0,0 +1,131 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+import FetchStore from "../src/fetch.js";
+
+// `vitest --api` exposes the port 51204
+// ref: https://vitest.dev/config/#api
+let href = "http://localhost:51204/fixtures/v3/data.zarr";
+
+describe("FetchStore", () => {
+	afterEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	it("reads a file from string url", async () => {
+		let store = new FetchStore(href);
+		let bytes = await store.get("/zarr.json");
+		expect(bytes).toBeInstanceOf(Uint8Array);
+		expect(JSON.parse(new TextDecoder().decode(bytes))).toMatchInlineSnapshot(`
+			{
+			  "attributes": {},
+			  "node_type": "group",
+			  "zarr_format": 3,
+			}
+		`);
+	});
+
+	it("reads a file from URL", async () => {
+		let store = new FetchStore(new URL(href));
+		let bytes = await store.get("/zarr.json");
+		expect(bytes).toBeInstanceOf(Uint8Array);
+		expect(JSON.parse(new TextDecoder().decode(bytes))).toMatchInlineSnapshot(`
+			{
+			  "attributes": {},
+			  "node_type": "group",
+			  "zarr_format": 3,
+			}
+		`);
+	});
+
+	it("reads multi-part path", async () => {
+		let store = new FetchStore(href);
+		let bytes = await store.get("/1d.chunked.i2/zarr.json");
+		expect(bytes).toBeInstanceOf(Uint8Array);
+		expect(JSON.parse(new TextDecoder().decode(bytes))).toMatchInlineSnapshot(`
+			{
+			  "attributes": {},
+			  "chunk_grid": {
+			    "configuration": {
+			      "chunk_shape": [
+			        2,
+			      ],
+			    },
+			    "name": "regular",
+			  },
+			  "chunk_key_encoding": {
+			    "configuration": {
+			      "separator": "/",
+			    },
+			    "name": "default",
+			  },
+			  "codecs": [
+			    {
+			      "configuration": {
+			        "endian": "little",
+			      },
+			      "name": "endian",
+			    },
+			    {
+			      "configuration": {
+			        "blocksize": 0,
+			        "clevel": 5,
+			        "cname": "zstd",
+			        "shuffle": "noshuffle",
+			        "typesize": 4,
+			      },
+			      "name": "blosc",
+			    },
+			  ],
+			  "data_type": "int16",
+			  "dimension_names": null,
+			  "fill_value": 0,
+			  "node_type": "array",
+			  "shape": [
+			    4,
+			  ],
+			  "zarr_format": 3,
+			}
+		`);
+	});
+
+	it("returns undefined for missing file", async () => {
+		let store = new FetchStore(href);
+		expect(await store.get("/missing.json")).toBeUndefined();
+	});
+
+	it("forwards request options to fetch", async () => {
+		let headers = { "x-test": "test" };
+		let store = new FetchStore(href);
+		let spy = vi.spyOn(globalThis, "fetch");
+		await store.get("/zarr.json", { headers });
+		expect(spy).toHaveBeenCalledWith(href + "/zarr.json", { headers });
+	});
+
+	it("forwards request options to fetch when configured globally", async () => {
+		let headers = { "x-test": "test" };
+		let store = new FetchStore(href, { headers });
+		let spy = vi.spyOn(globalThis, "fetch");
+		await store.get("/zarr.json");
+		expect(spy).toHaveBeenCalledWith(href + "/zarr.json", { headers });
+	});
+
+	it("overrides request options", async () => {
+		let opts: RequestInit = {
+			headers: { "x-test": "root", "x-test2": "root" },
+			cache: "no-cache",
+		};
+		let store = new FetchStore(href, opts);
+		let spy = vi.spyOn(globalThis, "fetch");
+		await store.get("/zarr.json", { headers: { "x-test": "override" } });
+		expect(spy).toHaveBeenCalledWith(href + "/zarr.json", {
+			headers: { "x-test": "override" },
+			cache: "no-cache",
+		});
+	});
+
+	it("checks if key exists", async () => {
+		let store = new FetchStore(href);
+		expect(await store.has("/zarr.json")).toBe(true);
+		expect(await store.has("/missing.json")).toBe(false);
+	});
+});

packages/storage/__tests__/fs.test.ts

@@ -0,0 +1,55 @@
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import * as url from "node:url";
+
+import FileSystemStore from "../src/fs.js";
+
+const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
+const store_path = path.join(__dirname, "teststore");
+
+beforeAll(async () => {
+	await fs.mkdir(store_path);
+});
+
+afterAll(async () => {
+	await fs.rm(store_path, { recursive: true });
+});
+
+describe("FileSystemStore", () => {
+	it("writes a file", async () => {
+		const store = new FileSystemStore(store_path);
+		await store.set("/foo-write", new TextEncoder().encode("bar"));
+		expect(
+			await fs.readFile(path.join(store_path, "foo-write"), "utf-8"),
+		).toBe("bar");
+	});
+	it("reads a file", async () => {
+		const store = new FileSystemStore(store_path);
+		await fs.writeFile(path.join(store_path, "foo-read"), "bar");
+		expect(
+			await store.get("/foo-read").then((buf) => new TextDecoder().decode(buf)),
+		).toBe("bar");
+	});
+	it("returns undefined for a non-existent file", async () => {
+		const store = new FileSystemStore(store_path);
+		expect(await store.get("/foo-does-not-exist")).toBe(undefined);
+	});
+	it("deletes a file", async () => {
+		const store = new FileSystemStore(store_path);
+		await fs.writeFile(path.join(store_path, "foo-delete"), "bar");
+		await store.delete("/foo-delete");
+		expect(
+			await fs
+				.readFile(path.join(store_path, "foo-delete"), "utf-8")
+				.catch((err) => err.code),
+		).toBe("ENOENT");
+	});
+	it("checks if a file exists", async () => {
+		const store = new FileSystemStore(store_path);
+		await fs.writeFile(path.join(store_path, "foo-exists"), "bar");
+		expect(await store.has("/foo-exists")).toBe(true);
+		expect(await store.has("/foo-does-not-exist")).toBe(false);
+	});
+});

packages/storage/package.json

@@ -9,12 +9,12 @@
 	},
 	"exports": {
 		".": {
-			"types": "./dist/index.d.ts",
-			"import": "./dist/index.js"
+			"types": "./dist/src/index.d.ts",
+			"import": "./dist/src/index.js"
 		},
 		"./*": {
-			"types": "./dist/*.d.ts",
-			"import": "./dist/*.js"
+			"types": "./dist/src/*.d.ts",
+			"import": "./dist/src/*.js"
 		}
 	},
 	"dependencies": {

packages/storage/src/fetch.ts

@@ -17,20 +17,23 @@ function resolve(root: string | URL, path: AbsolutePath): URL {
  * Must polyfill `fetch` for use in Node.js.
  *
  * ```typescript
- * import * as zarr from "zarrita/v2";
+ * import * as zarr from "@zarrita/core";
  * const store = new FetchStore("http://localhost:8080/data.zarr");
- * const arr = await zarr.get_array(store);
+ * const arr = await zarr.get(store, { kind: "array" });
  * ```
  */
 class FetchStore implements Async<Readable<RequestInit>> {
-	constructor(public url: string | URL) {}
+	constructor(
+		public url: string | URL,
+		public options: RequestInit = {},
+	) {}
 
 	async get(
 		key: AbsolutePath,
 		opts: RequestInit = {},
 	): Promise<Uint8Array | undefined> {
 		const { href } = resolve(this.url, key);
-		const res = await fetch(href, opts);
+		const res = await fetch(href, { ...this.options, ...opts });
 		if (res.status === 404 || res.status === 403) {
 			return undefined;
 		}
@@ -39,7 +42,7 @@ class FetchStore implements Async<Readable<RequestInit>> {
 	}
 
 	has(key: AbsolutePath): Promise<boolean> {
-		// TODO: make parameter, use HEAD request if possible.
+		// TODO: make parameter, use HEAD request if possible?
 		return this.get(key).then((res) => res !== undefined);
 	}
 }

packages/storage/src/ref.ts

@@ -2,23 +2,31 @@ import { parse } from "reference-spec-reader";
 import { fetch_range, strip_prefix, uri2href } from "./util.js";
 import type { AbsolutePath, Async, Readable } from "./types.js";
 
+type ReferenceEntry = string | [url: string | null] | [
+	url: string | null,
+	offset: number,
+	length: number,
+];
+
 interface ReferenceStoreOptions {
 	target?: string | URL;
 }
 
 /** @experimental */
 class ReferenceStore implements Async<Readable<RequestInit>> {
-	private target?: string | URL;
+	#refs: Map<string, ReferenceEntry>;
+	#opts: ReferenceStoreOptions;
 
 	constructor(
-		private refs: ReturnType<typeof parse>,
+		refs: Map<string, ReferenceEntry>,
 		opts: ReferenceStoreOptions = {},
 	) {
-		this.target = opts.target;
+		this.#refs = refs;
+		this.#opts = opts;
 	}
 
 	async get(key: AbsolutePath, opts: RequestInit = {}) {
-		let ref = this.refs.get(strip_prefix(key));
+		let ref = this.#refs.get(strip_prefix(key));
 
 		if (!ref) return;
 
@@ -30,7 +38,7 @@ class ReferenceStore implements Async<Readable<RequestInit>> {
 		}
 
 		let [urlOrNull, offset, size] = ref;
-		let url = urlOrNull ?? this.target;
+		let url = urlOrNull ?? this.#opts.target;
 		if (!url) {
 			throw Error(`No url for key ${key}, and no target url provided.`);
 		}
@@ -47,16 +55,16 @@ class ReferenceStore implements Async<Readable<RequestInit>> {
 	}
 
 	async has(key: AbsolutePath) {
-		return this.refs.has(strip_prefix(key));
+		return this.#refs.has(strip_prefix(key));
 	}
 
-	static fromSpec(spec: Record<string, any>, opts: ReferenceStoreOptions = {}) {
+	static fromSpec(spec: Record<string, any>, opts?: ReferenceStoreOptions) {
 		let refs = parse(spec);
 		return new ReferenceStore(refs, opts);
 	}
 
-	static async fromUrl(refUrl: string | URL, opts: ReferenceStoreOptions = {}) {
-		let spec = await fetch(refUrl as string).then((res) => res.json());
+	static async fromUrl(url: string | URL, opts?: ReferenceStoreOptions) {
+		let spec = await fetch(url).then((res) => res.json());
 		return ReferenceStore.fromSpec(spec, opts);
 	}
 }