Skip to content

Commit b90fd33

Browse files
manztmanzt
authored
feat(storage): support partial reads (#93)
1 parent 6e7df4f commit b90fd33

File tree

17 files changed

+297
-139
lines changed

17 files changed

+297
-139
lines changed

‎.changeset/young-schools-cry.md

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
"@zarrita/storage": patch
3+
---
4+
5+
feat: Support partial reads from `Readable`
6+
7+
Introduces the `Readable.getRange` method, which can be optionally implemented by a store to support partial reads.
8+
The `RangeQuery` param is inspired by the HTTP [`Range` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range).
9+
Allowing the `suffixLength` query means the store can decide the best way to return the final N bytes from a file.
10+
11+
```javascript
12+
const store = new FetchStore("http://localhost:8080/data.zarr");
13+
await store.getRange("/foo.json", { suffixLength: 100 });
14+
await store.getRange("/foo.json", { offset: 10, length: 20 });
15+
```

‎packages/core/src/consolidated.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { AbsolutePath, Async, Readable } from "@zarrita/storage";
1+
import type { AbsolutePath, Readable } from "@zarrita/storage";
22

33
import { Array, Group, Location } from "./hierarchy.js";
44
import {
@@ -15,7 +15,7 @@ type ConsolidatedMetadata = {
1515
};
1616

1717
async function get_consolidated_metadata(
18-
store: Async<Readable>,
18+
store: Readable,
1919
): Promise<ConsolidatedMetadata> {
2020
let bytes = await store.get("/.zmetadata");
2121
if (!bytes) throw new Error("No consolidated metadata found.");
@@ -27,7 +27,7 @@ async function get_consolidated_metadata(
2727
}
2828

2929
/** Proxies requests to the underlying store. */
30-
export async function openConsolidated<Store extends Async<Readable>>(
30+
export async function openConsolidated<Store extends Readable>(
3131
store: Store,
3232
) {
3333
let { metadata } = await get_consolidated_metadata(store);
@@ -72,7 +72,7 @@ export async function openConsolidated<Store extends Async<Readable>>(
7272
return new ConsolidatedHierarchy(nodes);
7373
}
7474

75-
class ConsolidatedHierarchy<Store extends Readable | Async<Readable>> {
75+
class ConsolidatedHierarchy<Store extends Readable> {
7676
constructor(
7777
public contents: Map<AbsolutePath, Array<DataType, Store> | Group<Store>>,
7878
) {}

‎packages/core/src/create.ts

+7-9
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { Async, Readable, Writeable } from "@zarrita/storage";
1+
import type { Mutable } from "@zarrita/storage";
22

33
import type {
44
ArrayMetadata,
@@ -26,30 +26,30 @@ interface CreateArrayOptions<Dtype extends DataType> {
2626
}
2727

2828
export async function create<
29-
Store extends (Readable & Writeable) | Async<Readable & Writeable>,
29+
Store extends Mutable,
3030
Dtype extends DataType = DataType,
3131
>(
3232
location: Location<Store> | Store,
3333
): Promise<Group<Store>>;
3434

3535
export async function create<
36-
Store extends (Readable & Writeable) | Async<Readable & Writeable>,
36+
Store extends Mutable,
3737
Dtype extends DataType = DataType,
3838
>(
3939
location: Location<Store> | Store,
4040
options: CreateGroupOptions,
4141
): Promise<Group<Store>>;
4242

4343
export async function create<
44-
Store extends (Readable & Writeable) | Async<Readable & Writeable>,
44+
Store extends Mutable,
4545
Dtype extends DataType,
4646
>(
4747
location: Location<Store> | Store,
4848
options: CreateArrayOptions<Dtype>,
4949
): Promise<Array<Dtype, Store>>;
5050

5151
export async function create<
52-
Store extends (Readable & Writeable) | Async<Readable & Writeable>,
52+
Store extends Mutable,
5353
Dtype extends DataType,
5454
>(
5555
location: Location<Store> | Store,
@@ -60,9 +60,7 @@ export async function create<
6060
return create_group(loc, options);
6161
}
6262

63-
async function create_group<
64-
Store extends (Readable & Writeable) | Async<Readable & Writeable>,
65-
>(
63+
async function create_group<Store extends Mutable>(
6664
location: Location<Store>,
6765
options: CreateGroupOptions = {},
6866
): Promise<Group<Store>> {
@@ -79,7 +77,7 @@ async function create_group<
7977
}
8078

8179
async function create_array<
82-
Store extends (Readable & Writeable) | Async<Readable & Writeable>,
80+
Store extends Mutable,
8381
Dtype extends DataType,
8482
>(
8583
location: Location<Store>,

‎packages/core/src/hierarchy.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { AbsolutePath, Async, Readable } from "@zarrita/storage";
1+
import type { AbsolutePath, Readable } from "@zarrita/storage";
22
import type {
33
ArrayMetadata,
44
Chunk,
@@ -45,7 +45,7 @@ export function root<Store>(
4545
}
4646

4747
export class Group<
48-
Store extends Readable | Async<Readable> = Readable | Async<Readable>,
48+
Store extends Readable,
4949
> extends Location<Store> {
5050
readonly kind = "group";
5151
#metadata: GroupMetadata;
@@ -99,7 +99,7 @@ interface ArrayContext<D extends DataType> {
9999

100100
export class Array<
101101
Dtype extends DataType,
102-
Store extends Readable | Async<Readable> = Readable | Async<Readable>,
102+
Store extends Readable = Readable,
103103
> extends Location<Store> {
104104
readonly kind = "array";
105105
#metadata: ArrayMetadata<Dtype>;

‎packages/core/src/open.ts

+22-18
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { Async, Readable } from "@zarrita/storage";
1+
import type { Readable } from "@zarrita/storage";
22
import type {
33
ArrayMetadata,
44
Attributes,
@@ -14,29 +14,29 @@ import {
1414
} from "./util.js";
1515

1616
async function load_attrs(
17-
location: Location<Readable | Async<Readable>>,
17+
location: Location<Readable>,
1818
): Promise<Attributes> {
1919
let meta_bytes = await location.store.get(location.resolve(".zattrs").path);
2020
if (!meta_bytes) return {};
2121
return json_decode_object(meta_bytes);
2222
}
2323

24-
function open_v2<Store extends Readable | Async<Readable>>(
24+
function open_v2<Store extends Readable>(
2525
location: Location<Store> | Store,
2626
options: { kind: "group"; attrs?: boolean },
2727
): Promise<Group<Store>>;
2828

29-
function open_v2<Store extends Readable | Async<Readable>>(
29+
function open_v2<Store extends Readable>(
3030
location: Location<Store> | Store,
3131
options: { kind: "array"; attrs?: boolean },
3232
): Promise<Array<DataType, Store>>;
3333

34-
function open_v2<Store extends Readable | Async<Readable>>(
34+
function open_v2<Store extends Readable>(
3535
location: Location<Store> | Store,
3636
options?: { kind?: "array" | "group"; attrs?: boolean },
3737
): Promise<Array<DataType, Store> | Group<Store>>;
3838

39-
async function open_v2<Store extends Readable | Async<Readable>>(
39+
async function open_v2<Store extends Readable>(
4040
location: Location<Store> | Store,
4141
options: { kind?: "array" | "group"; attrs?: boolean } = {},
4242
) {
@@ -51,7 +51,7 @@ async function open_v2<Store extends Readable | Async<Readable>>(
5151
});
5252
}
5353

54-
async function open_array_v2<Store extends Readable | Async<Readable>>(
54+
async function open_array_v2<Store extends Readable>(
5555
location: Location<Store>,
5656
attrs: Attributes,
5757
) {
@@ -67,7 +67,7 @@ async function open_array_v2<Store extends Readable | Async<Readable>>(
6767
);
6868
}
6969

70-
async function open_group_v2<Store extends Readable | Async<Readable>>(
70+
async function open_group_v2<Store extends Readable>(
7171
location: Location<Store>,
7272
attrs: Attributes,
7373
) {
@@ -83,7 +83,7 @@ async function open_group_v2<Store extends Readable | Async<Readable>>(
8383
);
8484
}
8585

86-
async function _open_v3<Store extends Readable | Async<Readable>>(
86+
async function _open_v3<Store extends Readable>(
8787
location: Location<Store>,
8888
) {
8989
let { store, path } = location.resolve("zarr.json");
@@ -99,21 +99,25 @@ async function _open_v3<Store extends Readable | Async<Readable>>(
9999
: new Group(store, location.path, meta_doc);
100100
}
101101

102-
function open_v3<Store extends Readable | Async<Readable>>(
102+
function open_v3<Store extends Readable>(
103103
location: Location<Store> | Store,
104104
options: { kind: "group" },
105105
): Promise<Group<Store>>;
106106

107-
function open_v3<Store extends Readable | Async<Readable>>(
107+
function open_v3<Store extends Readable>(
108108
location: Location<Store> | Store,
109109
options: { kind: "array" },
110110
): Promise<Array<DataType, Store>>;
111111

112-
function open_v3<Store extends Readable | Async<Readable>>(
112+
function open_v3<Store extends Readable>(
113113
location: Location<Store> | Store,
114114
): Promise<Array<DataType, Store> | Group<Store>>;
115115

116-
async function open_v3<Store extends Readable | Async<Readable>>(
116+
function open_v3<Store extends Readable>(
117+
location: Location<Store> | Store,
118+
): Promise<Array<DataType, Store> | Group<Store>>;
119+
120+
async function open_v3<Store extends Readable>(
117121
location: Location<Store>,
118122
options: { kind?: "array" | "group" } = {},
119123
): Promise<Array<DataType, Store> | Group<Store>> {
@@ -126,26 +130,26 @@ async function open_v3<Store extends Readable | Async<Readable>>(
126130
throw new Error(`Expected node of kind ${options.kind}, found ${kind}.`);
127131
}
128132

129-
export function open<Store extends Readable | Async<Readable>>(
133+
export function open<Store extends Readable>(
130134
location: Location<Store> | Store,
131135
options: { kind: "group" },
132136
): Promise<Group<Store>>;
133137

134-
export function open<Store extends Readable | Async<Readable>>(
138+
export function open<Store extends Readable>(
135139
location: Location<Store> | Store,
136140
options: { kind: "array" },
137141
): Promise<Array<DataType, Store>>;
138142

139-
export function open<Store extends Readable | Async<Readable>>(
143+
export function open<Store extends Readable>(
140144
location: Location<Store> | Store,
141145
options: { kind: "auto" },
142146
): Promise<Array<DataType, Store> | Group<Store>>;
143147

144-
export function open<Store extends Readable | Async<Readable>>(
148+
export function open<Store extends Readable>(
145149
location: Location<Store> | Store,
146150
): Promise<Array<DataType, Store> | Group<Store>>;
147151

148-
export async function open<Store extends Readable | Async<Readable>>(
152+
export async function open<Store extends Readable>(
149153
location: Location<Store> | Store,
150154
options: { kind: "auto" | "array" | "group" } = { kind: "auto" },
151155
): Promise<Array<DataType, Store> | Group<Store>> {

‎packages/indexing/src/get.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { Async, Readable } from "@zarrita/storage";
1+
import type { Readable } from "@zarrita/storage";
22
import type { Array, Chunk, DataType, Scalar, TypedArray } from "@zarrita/core";
33
import type {
44
GetOptions,
@@ -21,7 +21,7 @@ function unwrap<D extends DataType>(
2121

2222
export async function get<
2323
D extends DataType,
24-
Store extends Readable | Async<Readable>,
24+
Store extends Readable,
2525
Arr extends Chunk<D>,
2626
Sel extends (null | Slice | number)[],
2727
>(

‎packages/indexing/src/ops.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { Async, Readable, Writeable } from "@zarrita/storage";
1+
import type { Mutable, Readable } from "@zarrita/storage";
22
import type * as core from "@zarrita/core";
33
import {
44
BoolArray,
@@ -129,7 +129,7 @@ export const setter = {
129129
/** @category Utility */
130130
export async function get<
131131
D extends core.DataType,
132-
Store extends Readable | Async<Readable>,
132+
Store extends Readable,
133133
Sel extends (null | Slice | number)[],
134134
>(
135135
arr: core.Array<D, Store>,
@@ -148,7 +148,7 @@ export async function get<
148148
export async function set<
149149
D extends core.DataType,
150150
>(
151-
arr: core.Array<D, (Readable & Writeable) | Async<Readable & Writeable>>,
151+
arr: core.Array<D, Mutable>,
152152
selection: (null | Slice | number)[] | null,
153153
value: core.Scalar<D> | core.Chunk<D>,
154154
opts: SetOptions = {},

‎packages/indexing/src/set.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { _internal_get_array_context, KeyError } from "@zarrita/core";
2-
import type { Async, Readable, Writeable } from "@zarrita/storage";
2+
import type { Mutable } from "@zarrita/storage";
33
import type { Array, Chunk, DataType, Scalar, TypedArray } from "@zarrita/core";
44

55
import { create_queue } from "./util.js";
@@ -19,7 +19,7 @@ function flip_indexer_projection(m: IndexerProjection) {
1919
}
2020

2121
export async function set<Dtype extends DataType, Arr extends Chunk<Dtype>>(
22-
arr: Array<Dtype, (Readable & Writeable) | Async<Readable & Writeable>>,
22+
arr: Array<Dtype, Mutable>,
2323
selection: (number | Slice | null)[] | null,
2424
value: Scalar<Dtype> | Arr,
2525
opts: SetOptions,

‎packages/ndarray/index.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import type {
1010
Slice,
1111
} from "@zarrita/indexing";
1212
import type * as core from "@zarrita/core";
13-
import type { Async, Readable, Writeable } from "@zarrita/storage";
13+
import type { Mutable, Readable } from "@zarrita/storage";
1414

1515
export const setter = {
1616
prepare: ndarray,
@@ -34,7 +34,7 @@ export const setter = {
3434
/** @category Utility */
3535
export async function get<
3636
D extends core.DataType,
37-
Store extends Readable | Async<Readable>,
37+
Store extends Readable,
3838
Sel extends (null | Slice | number)[],
3939
>(
4040
arr: core.Array<D, Store>,
@@ -51,7 +51,7 @@ export async function get<
5151

5252
/** @category Utility */
5353
export async function set<D extends core.DataType>(
54-
arr: core.Array<D, (Readable & Writeable) | Async<Readable & Writeable>>,
54+
arr: core.Array<D, Mutable>,
5555
selection: (null | Slice | number)[] | null,
5656
value: core.Scalar<D> | ndarray.NdArray<core.TypedArray<D>>,
5757
opts: SetOptions = {},

‎packages/storage/__tests__/fetch.test.ts

+18-8
Original file line numberDiff line numberDiff line change
@@ -103,29 +103,39 @@ describe("FetchStore", () => {
103103

104104
it("forwards request options to fetch when configured globally", async () => {
105105
let headers = { "x-test": "test" };
106-
let store = new FetchStore(href, { headers });
106+
let store = new FetchStore(href, { overrides: { headers } });
107107
let spy = vi.spyOn(globalThis, "fetch");
108108
await store.get("/zarr.json");
109109
expect(spy).toHaveBeenCalledWith(href + "/zarr.json", { headers });
110110
});
111111

112-
it("overrides request options", async () => {
113-
let opts: RequestInit = {
112+
it("merges request options", async () => {
113+
let overrides: RequestInit = {
114114
headers: { "x-test": "root", "x-test2": "root" },
115115
cache: "no-cache",
116116
};
117-
let store = new FetchStore(href, opts);
117+
let store = new FetchStore(href, { overrides });
118118
let spy = vi.spyOn(globalThis, "fetch");
119119
await store.get("/zarr.json", { headers: { "x-test": "override" } });
120120
expect(spy).toHaveBeenCalledWith(href + "/zarr.json", {
121-
headers: { "x-test": "override" },
121+
headers: { "x-test": "override", "x-test2": "root" },
122122
cache: "no-cache",
123123
});
124124
});
125125

126-
it("checks if key exists", async () => {
126+
it("reads partial - suffixLength", async () => {
127127
let store = new FetchStore(href);
128-
expect(await store.has("/zarr.json")).toBe(true);
129-
expect(await store.has("/missing.json")).toBe(false);
128+
let bytes = await store.getRange("/zarr.json", { suffixLength: 50 });
129+
expect(new TextDecoder().decode(bytes)).toMatchInlineSnapshot(
130+
'"utes\\": {}, \\"zarr_format\\": 3, \\"node_type\\": \\"group\\"}"',
131+
);
132+
});
133+
134+
it("reads partial - offset, length", async () => {
135+
let store = new FetchStore(href);
136+
let bytes = await store.getRange("/zarr.json", { offset: 4, length: 50 });
137+
expect(new TextDecoder().decode(bytes)).toMatchInlineSnapshot(
138+
'"tributes\\": {}, \\"zarr_format\\": 3, \\"node_type\\": \\"gro"',
139+
);
130140
});
131141
});

0 commit comments

Comments
 (0)