feat(storage): support partial reads

.changeset/young-schools-cry.md

@@ -0,0 +1,15 @@
+---
+"@zarrita/storage": patch
+---
+
+feat: Support partial reads from `Readable`
+
+Introduces the `Readable.getRange` method, which can be optionally implemented by a store to support partial reads.
+The `RangeQuery` param is inspired by the HTTP [`Range` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range).
+Allowing the `suffixLength` query means the store can decide the best way to return the final N bytes from a file.
+
+```javascript
+const store = new FetchStore("http://localhost:8080/data.zarr");
+await store.getRange("/foo.json", { suffixLength: 100 });
+await store.getRange("/foo.json", { offset: 10, length: 20 });
+```

packages/core/src/consolidated.ts

@@ -1,4 +1,4 @@
-import type { AbsolutePath, Async, Readable } from "@zarrita/storage";
+import type { AbsolutePath, Readable } from "@zarrita/storage";
 
 import { Array, Group, Location } from "./hierarchy.js";
 import {
@@ -15,7 +15,7 @@ type ConsolidatedMetadata = {
 };
 
 async function get_consolidated_metadata(
-	store: Async<Readable>,
+	store: Readable,
 ): Promise<ConsolidatedMetadata> {
 	let bytes = await store.get("/.zmetadata");
 	if (!bytes) throw new Error("No consolidated metadata found.");
@@ -27,7 +27,7 @@ async function get_consolidated_metadata(
 }
 
 /** Proxies requests to the underlying store. */
-export async function openConsolidated<Store extends Async<Readable>>(
+export async function openConsolidated<Store extends Readable>(
 	store: Store,
 ) {
 	let { metadata } = await get_consolidated_metadata(store);
@@ -72,7 +72,7 @@ export async function openConsolidated<Store extends Async<Readable>>(
 	return new ConsolidatedHierarchy(nodes);
 }
 
-class ConsolidatedHierarchy<Store extends Readable | Async<Readable>> {
+class ConsolidatedHierarchy<Store extends Readable> {
 	constructor(
 		public contents: Map<AbsolutePath, Array<DataType, Store> | Group<Store>>,
 	) {}

packages/core/src/create.ts

@@ -1,4 +1,4 @@
-import type { Async, Readable, Writeable } from "@zarrita/storage";
+import type { Mutable } from "@zarrita/storage";
 
 import type {
 	ArrayMetadata,
@@ -26,30 +26,30 @@ interface CreateArrayOptions<Dtype extends DataType> {
 }
 
 export async function create<
-	Store extends (Readable & Writeable) | Async<Readable & Writeable>,
+	Store extends Mutable,
 	Dtype extends DataType = DataType,
 >(
 	location: Location<Store> | Store,
 ): Promise<Group<Store>>;
 
 export async function create<
-	Store extends (Readable & Writeable) | Async<Readable & Writeable>,
+	Store extends Mutable,
 	Dtype extends DataType = DataType,
 >(
 	location: Location<Store> | Store,
 	options: CreateGroupOptions,
 ): Promise<Group<Store>>;
 
 export async function create<
-	Store extends (Readable & Writeable) | Async<Readable & Writeable>,
+	Store extends Mutable,
 	Dtype extends DataType,
 >(
 	location: Location<Store> | Store,
 	options: CreateArrayOptions<Dtype>,
 ): Promise<Array<Dtype, Store>>;
 
 export async function create<
-	Store extends (Readable & Writeable) | Async<Readable & Writeable>,
+	Store extends Mutable,
 	Dtype extends DataType,
 >(
 	location: Location<Store> | Store,
@@ -60,9 +60,7 @@ export async function create<
 	return create_group(loc, options);
 }
 
-async function create_group<
-	Store extends (Readable & Writeable) | Async<Readable & Writeable>,
->(
+async function create_group<Store extends Mutable>(
 	location: Location<Store>,
 	options: CreateGroupOptions = {},
 ): Promise<Group<Store>> {
@@ -79,7 +77,7 @@ async function create_group<
 }
 
 async function create_array<
-	Store extends (Readable & Writeable) | Async<Readable & Writeable>,
+	Store extends Mutable,
 	Dtype extends DataType,
 >(
 	location: Location<Store>,

packages/core/src/hierarchy.ts

@@ -1,4 +1,4 @@
-import type { AbsolutePath, Async, Readable } from "@zarrita/storage";
+import type { AbsolutePath, Readable } from "@zarrita/storage";
 import type {
 	ArrayMetadata,
 	Chunk,
@@ -45,7 +45,7 @@ export function root<Store>(
 }
 
 export class Group<
-	Store extends Readable | Async<Readable> = Readable | Async<Readable>,
+	Store extends Readable,
 > extends Location<Store> {
 	readonly kind = "group";
 	#metadata: GroupMetadata;
@@ -99,7 +99,7 @@ interface ArrayContext<D extends DataType> {
 
 export class Array<
 	Dtype extends DataType,
-	Store extends Readable | Async<Readable> = Readable | Async<Readable>,
+	Store extends Readable = Readable,
 > extends Location<Store> {
 	readonly kind = "array";
 	#metadata: ArrayMetadata<Dtype>;

packages/core/src/open.ts

@@ -1,4 +1,4 @@
-import type { Async, Readable } from "@zarrita/storage";
+import type { Readable } from "@zarrita/storage";
 import type {
 	ArrayMetadata,
 	Attributes,
@@ -14,29 +14,29 @@ import {
 } from "./util.js";
 
 async function load_attrs(
-	location: Location<Readable | Async<Readable>>,
+	location: Location<Readable>,
 ): Promise<Attributes> {
 	let meta_bytes = await location.store.get(location.resolve(".zattrs").path);
 	if (!meta_bytes) return {};
 	return json_decode_object(meta_bytes);
 }
 
-function open_v2<Store extends Readable | Async<Readable>>(
+function open_v2<Store extends Readable>(
 	location: Location<Store> | Store,
 	options: { kind: "group"; attrs?: boolean },
 ): Promise<Group<Store>>;
 
-function open_v2<Store extends Readable | Async<Readable>>(
+function open_v2<Store extends Readable>(
 	location: Location<Store> | Store,
 	options: { kind: "array"; attrs?: boolean },
 ): Promise<Array<DataType, Store>>;
 
-function open_v2<Store extends Readable | Async<Readable>>(
+function open_v2<Store extends Readable>(
 	location: Location<Store> | Store,
 	options?: { kind?: "array" | "group"; attrs?: boolean },
 ): Promise<Array<DataType, Store> | Group<Store>>;
 
-async function open_v2<Store extends Readable | Async<Readable>>(
+async function open_v2<Store extends Readable>(
 	location: Location<Store> | Store,
 	options: { kind?: "array" | "group"; attrs?: boolean } = {},
 ) {
@@ -51,7 +51,7 @@ async function open_v2<Store extends Readable | Async<Readable>>(
 	});
 }
 
-async function open_array_v2<Store extends Readable | Async<Readable>>(
+async function open_array_v2<Store extends Readable>(
 	location: Location<Store>,
 	attrs: Attributes,
 ) {
@@ -67,7 +67,7 @@ async function open_array_v2<Store extends Readable | Async<Readable>>(
 	);
 }
 
-async function open_group_v2<Store extends Readable | Async<Readable>>(
+async function open_group_v2<Store extends Readable>(
 	location: Location<Store>,
 	attrs: Attributes,
 ) {
@@ -83,7 +83,7 @@ async function open_group_v2<Store extends Readable | Async<Readable>>(
 	);
 }
 
-async function _open_v3<Store extends Readable | Async<Readable>>(
+async function _open_v3<Store extends Readable>(
 	location: Location<Store>,
 ) {
 	let { store, path } = location.resolve("zarr.json");
@@ -99,21 +99,25 @@ async function _open_v3<Store extends Readable | Async<Readable>>(
 		: new Group(store, location.path, meta_doc);
 }
 
-function open_v3<Store extends Readable | Async<Readable>>(
+function open_v3<Store extends Readable>(
 	location: Location<Store> | Store,
 	options: { kind: "group" },
 ): Promise<Group<Store>>;
 
-function open_v3<Store extends Readable | Async<Readable>>(
+function open_v3<Store extends Readable>(
 	location: Location<Store> | Store,
 	options: { kind: "array" },
 ): Promise<Array<DataType, Store>>;
 
-function open_v3<Store extends Readable | Async<Readable>>(
+function open_v3<Store extends Readable>(
 	location: Location<Store> | Store,
 ): Promise<Array<DataType, Store> | Group<Store>>;
 
-async function open_v3<Store extends Readable | Async<Readable>>(
+function open_v3<Store extends Readable>(
+	location: Location<Store> | Store,
+): Promise<Array<DataType, Store> | Group<Store>>;
+
+async function open_v3<Store extends Readable>(
 	location: Location<Store>,
 	options: { kind?: "array" | "group" } = {},
 ): Promise<Array<DataType, Store> | Group<Store>> {
@@ -126,26 +130,26 @@ async function open_v3<Store extends Readable | Async<Readable>>(
 	throw new Error(`Expected node of kind ${options.kind}, found ${kind}.`);
 }
 
-export function open<Store extends Readable | Async<Readable>>(
+export function open<Store extends Readable>(
 	location: Location<Store> | Store,
 	options: { kind: "group" },
 ): Promise<Group<Store>>;
 
-export function open<Store extends Readable | Async<Readable>>(
+export function open<Store extends Readable>(
 	location: Location<Store> | Store,
 	options: { kind: "array" },
 ): Promise<Array<DataType, Store>>;
 
-export function open<Store extends Readable | Async<Readable>>(
+export function open<Store extends Readable>(
 	location: Location<Store> | Store,
 	options: { kind: "auto" },
 ): Promise<Array<DataType, Store> | Group<Store>>;
 
-export function open<Store extends Readable | Async<Readable>>(
+export function open<Store extends Readable>(
 	location: Location<Store> | Store,
 ): Promise<Array<DataType, Store> | Group<Store>>;
 
-export async function open<Store extends Readable | Async<Readable>>(
+export async function open<Store extends Readable>(
 	location: Location<Store> | Store,
 	options: { kind: "auto" | "array" | "group" } = { kind: "auto" },
 ): Promise<Array<DataType, Store> | Group<Store>> {

packages/indexing/src/get.ts

@@ -1,4 +1,4 @@
-import type { Async, Readable } from "@zarrita/storage";
+import type { Readable } from "@zarrita/storage";
 import type { Array, Chunk, DataType, Scalar, TypedArray } from "@zarrita/core";
 import type {
 	GetOptions,
@@ -21,7 +21,7 @@ function unwrap<D extends DataType>(
 
 export async function get<
 	D extends DataType,
-	Store extends Readable | Async<Readable>,
+	Store extends Readable,
 	Arr extends Chunk<D>,
 	Sel extends (null | Slice | number)[],
 >(

packages/indexing/src/ops.ts

@@ -1,4 +1,4 @@
-import type { Async, Readable, Writeable } from "@zarrita/storage";
+import type { Mutable, Readable } from "@zarrita/storage";
 import type * as core from "@zarrita/core";
 import {
 	BoolArray,
@@ -129,7 +129,7 @@ export const setter = {
 /** @category Utility */
 export async function get<
 	D extends core.DataType,
-	Store extends Readable | Async<Readable>,
+	Store extends Readable,
 	Sel extends (null | Slice | number)[],
 >(
 	arr: core.Array<D, Store>,
@@ -148,7 +148,7 @@ export async function get<
 export async function set<
 	D extends core.DataType,
 >(
-	arr: core.Array<D, (Readable & Writeable) | Async<Readable & Writeable>>,
+	arr: core.Array<D, Mutable>,
 	selection: (null | Slice | number)[] | null,
 	value: core.Scalar<D> | core.Chunk<D>,
 	opts: SetOptions = {},

packages/indexing/src/set.ts

@@ -1,5 +1,5 @@
 import { _internal_get_array_context, KeyError } from "@zarrita/core";
-import type { Async, Readable, Writeable } from "@zarrita/storage";
+import type { Mutable } from "@zarrita/storage";
 import type { Array, Chunk, DataType, Scalar, TypedArray } from "@zarrita/core";
 
 import { create_queue } from "./util.js";
@@ -19,7 +19,7 @@ function flip_indexer_projection(m: IndexerProjection) {
 }
 
 export async function set<Dtype extends DataType, Arr extends Chunk<Dtype>>(
-	arr: Array<Dtype, (Readable & Writeable) | Async<Readable & Writeable>>,
+	arr: Array<Dtype, Mutable>,
 	selection: (number | Slice | null)[] | null,
 	value: Scalar<Dtype> | Arr,
 	opts: SetOptions,

packages/ndarray/index.ts

@@ -10,7 +10,7 @@ import type {
 	Slice,
 } from "@zarrita/indexing";
 import type * as core from "@zarrita/core";
-import type { Async, Readable, Writeable } from "@zarrita/storage";
+import type { Mutable, Readable } from "@zarrita/storage";
 
 export const setter = {
 	prepare: ndarray,
@@ -34,7 +34,7 @@ export const setter = {
 /** @category Utility */
 export async function get<
 	D extends core.DataType,
-	Store extends Readable | Async<Readable>,
+	Store extends Readable,
 	Sel extends (null | Slice | number)[],
 >(
 	arr: core.Array<D, Store>,
@@ -51,7 +51,7 @@ export async function get<
 
 /** @category Utility */
 export async function set<D extends core.DataType>(
-	arr: core.Array<D, (Readable & Writeable) | Async<Readable & Writeable>>,
+	arr: core.Array<D, Mutable>,
 	selection: (null | Slice | number)[] | null,
 	value: core.Scalar<D> | ndarray.NdArray<core.TypedArray<D>>,
 	opts: SetOptions = {},

packages/storage/__tests__/fetch.test.ts

@@ -103,29 +103,39 @@ describe("FetchStore", () => {
 
 	it("forwards request options to fetch when configured globally", async () => {
 		let headers = { "x-test": "test" };
-		let store = new FetchStore(href, { headers });
+		let store = new FetchStore(href, { overrides: { headers } });
 		let spy = vi.spyOn(globalThis, "fetch");
 		await store.get("/zarr.json");
 		expect(spy).toHaveBeenCalledWith(href + "/zarr.json", { headers });
 	});
 
-	it("overrides request options", async () => {
-		let opts: RequestInit = {
+	it("merges request options", async () => {
+		let overrides: RequestInit = {
 			headers: { "x-test": "root", "x-test2": "root" },
 			cache: "no-cache",
 		};
-		let store = new FetchStore(href, opts);
+		let store = new FetchStore(href, { overrides });
 		let spy = vi.spyOn(globalThis, "fetch");
 		await store.get("/zarr.json", { headers: { "x-test": "override" } });
 		expect(spy).toHaveBeenCalledWith(href + "/zarr.json", {
-			headers: { "x-test": "override" },
+			headers: { "x-test": "override", "x-test2": "root" },
 			cache: "no-cache",
 		});
 	});
 
-	it("checks if key exists", async () => {
+	it("reads partial - suffixLength", async () => {
 		let store = new FetchStore(href);
-		expect(await store.has("/zarr.json")).toBe(true);
-		expect(await store.has("/missing.json")).toBe(false);
+		let bytes = await store.getRange("/zarr.json", { suffixLength: 50 });
+		expect(new TextDecoder().decode(bytes)).toMatchInlineSnapshot(
+			'"utes\\": {}, \\"zarr_format\\": 3, \\"node_type\\": \\"group\\"}"',
+		);
+	});
+
+	it("reads partial - offset, length", async () => {
+		let store = new FetchStore(href);
+		let bytes = await store.getRange("/zarr.json", { offset: 4, length: 50 });
+		expect(new TextDecoder().decode(bytes)).toMatchInlineSnapshot(
+			'"tributes\\": {}, \\"zarr_format\\": 3, \\"node_type\\": \\"gro"',
+		);
 	});
 });