Conslidate @zarrita/indexing in to @zarrita/core

.changeset/evil-bees-start.md

@@ -0,0 +1,10 @@
+---
+"@zarrita/indexing": patch
+"@zarrita/ndarray": patch
+"zarrita": patch
+"@zarrita/core": patch
+---
+
+Conslidate `@zarrita/indexing` package into `@zarrita/core`
+
+Now `@zarrita/indexing` re-exports from `zarrita`. End users should prefer to import from `zarrita`.

demo.html

@@ -6,9 +6,8 @@
     <script type="importmap">
       {
         "imports": {
+          "zarrita": "./packages/zarrita/dist/index.js",
           "@zarrita/core": "./packages/core/dist/src/index.js",
-          "@zarrita/typedarray": "./packages/typedarray/dist/src/index.js",
-          "@zarrita/indexing": "./packages/indexing/dist/src/index.js",
           "@zarrita/storage/fetch": "./packages/storage/dist/src/fetch.js",
           "numcodecs/blosc": "./packages/core/node_modules/numcodecs/blosc.js",
           "numcodecs/lz4": "./packages/core/node_modules/numcodecs/lz4.js",
@@ -20,11 +19,8 @@
     </script>
   </head>
   <script type="module">
-    import * as zarr from "@zarrita/core";
-    import { get } from "@zarrita/indexing";
-    import FetchStore from "@zarrita/storage/fetch";
-
-    let store = zarr.root(new FetchStore(new URL("fixtures", import.meta.url)));
+    import * as zarr from "zarrita";
+    let store = zarr.root(new zarr.FetchStore(new URL("fixtures", import.meta.url)));
 
     {
       let grp = await zarr.open.v2(store.resolve("v2/data.zarr"), {
@@ -58,7 +54,7 @@
         "3d.contiguous.i2",
       ]) {
         let arr = await zarr.open.v2(grp.resolve(fixture), { kind: "array", attrs: false });
-        let { data, shape, stride } = await get(arr);
+        let { data, shape, stride } = await zarr.get(arr);
         let pre = document.createElement("pre");
         pre.textContent = `\
 {
@@ -111,7 +107,7 @@
         "3d.contiguous.i2",
       ]) {
         let arr = await zarr.open.v3(grp.resolve(fixture), { kind: "array" });
-        let { data, shape, stride } = await get(arr);
+        let { data, shape, stride } = await zarr.get(arr);
         let pre = document.createElement("pre");
         pre.textContent = `\
 {

packages/indexing/__tests__/get.test.ts → packages/core/__tests__/indexing/get.test.ts

@@ -1,20 +1,20 @@
 import * as path from "node:path";
 import * as url from "node:url";
-import * as zarr from "@zarrita/core";
-import FSStore from "@zarrita/storage/fs";
+import FileSystemStore from "@zarrita/storage/fs";
 import { describe, expect, it } from "vitest";
 
-import { get } from "../src/ops.js";
-import { range } from "../src/util.js";
+import * as zarr from "../../src/index.js";
+import { get } from "../../src/indexing/ops.js";
+import { range } from "../../src/indexing/util.js";
 
 let __dirname = path.dirname(url.fileURLToPath(import.meta.url));
 
 async function get_v2(
 	abs_path: string,
 	...args: unknown[]
 ): Promise<zarr.Chunk<zarr.DataType>> {
-	let root = path.resolve(__dirname, "../../../fixtures/v2/data.zarr");
-	let store = zarr.root(new FSStore(root));
+	let root = path.resolve(__dirname, "../../../../fixtures/v2/data.zarr");
+	let store = zarr.root(new FileSystemStore(root));
 	let arr = await zarr.open.v2(store.resolve(abs_path), { kind: "array" });
 	// @ts-expect-error - TS not happy about spreading these args and its fine for this func
 	return get(arr, ...args);
@@ -539,8 +539,8 @@ async function get_v3(
 	abs_path: string,
 	...args: unknown[]
 ): Promise<zarr.Chunk<zarr.DataType>> {
-	let root = path.resolve(__dirname, "../../../fixtures/v3/data.zarr");
-	let store = zarr.root(new FSStore(root));
+	let root = path.resolve(__dirname, "../../../../fixtures/v3/data.zarr");
+	let store = zarr.root(new FileSystemStore(root));
 	let arr = await zarr.open.v3(store.resolve(abs_path), { kind: "array" });
 	// @ts-expect-error - TS not happy about spreading these args
 	return get(arr, ...args);

packages/indexing/__tests__/indexer.test.ts → packages/core/__tests__/indexing/indexer.test.ts

@@ -1,11 +1,12 @@
 import { describe, expect, test } from "vitest";
+
 import {
 	BasicIndexer,
 	normalize_integer_selection,
 	normalize_selection,
-} from "../src/indexer.js";
-import type { Slice } from "../src/types.js";
-import { slice } from "../src/util.js";
+} from "../../src/indexing/indexer.js";
+import type { Slice } from "../../src/indexing/types.js";
+import { slice } from "../../src/indexing/util.js";
 
 describe("normalize_selection", () => {
 	// null !== null, so need custom compare

packages/indexing/__tests__/set.test.ts → packages/core/__tests__/indexing/set.test.ts

@@ -1,10 +1,9 @@
 import ndarray from "ndarray";
 import { expect, test } from "vitest";
 
-import * as zarr from "@zarrita/core";
-
-import { get, set } from "../src/ops.js";
-import { range, slice } from "../src/util.js";
+import * as zarr from "../../src/index.js";
+import { get, set } from "../../src/indexing/ops.js";
+import { range, slice } from "../../src/indexing/util.js";
 
 test("Read and write array data - builtin", async () => {
 	let h = zarr.root(new Map());
@@ -71,9 +70,9 @@ test("Read and write array data - builtin", async () => {
 	res = await get(a, [null, slice(0, 7)]);
 	expect(res.shape).toStrictEqual([5, 7]);
 	// biome-ignore format: the array should not be formatted
-	expect(res.data).toStrictEqual(new Int32Array([ 
-		 0,  1,  2,  3,  4,
-		 5,  6, 10, 11, 12,
+	expect(res.data).toStrictEqual(new Int32Array([
+		0, 1, 2, 3, 4,
+		5, 6, 10, 11, 12,
 		13, 14, 15, 16, 20,
 		21, 22, 23, 24, 25,
 		26, 30, 31, 32, 33,
@@ -85,15 +84,15 @@ test("Read and write array data - builtin", async () => {
 	expect(res.shape).toStrictEqual([3, 10]);
 	// biome-ignore format: the array should not be formatted
 	expect(res.data).toStrictEqual(new Int32Array([
-		 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,
+		0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
 		10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
 		20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
 	]));
 	res = await get(a, [slice(0, 3), slice(0, 7)]);
 	expect(res.shape).toStrictEqual([3, 7]);
 	// biome-ignore format: the array should not be formatted
 	expect(res.data).toStrictEqual(new Int32Array([
-		 0,  1,  2,  3,  4,  5,  6,
+		0, 1, 2, 3, 4, 5, 6,
 		10, 11, 12, 13, 14, 15, 16,
 		20, 21, 22, 23, 24, 25, 26,
 	]));

packages/indexing/__tests__/setter.test.ts → packages/core/__tests__/indexing/setter.test.ts

@@ -1,11 +1,11 @@
-import type { Chunk, DataType } from "@zarrita/core";
 import ndarray from "ndarray";
 import { assign } from "ndarray-ops";
 import { describe, expect, it } from "vitest";
 
-import { setter } from "../src/ops.js";
-import type { Projection } from "../src/types.js";
-import { slice, slice_indices } from "../src/util.js";
+import type * as zarr from "../../src/index.js";
+import { setter } from "../../src/indexing/ops.js";
+import type { Projection } from "../../src/indexing/types.js";
+import { slice, slice_indices } from "../../src/indexing/util.js";
 
 /** Compute strides for 'C' or 'F' ordered array from shape */
 function get_strides(shape: readonly number[], order: "C" | "F") {
@@ -32,7 +32,11 @@ function col_major_stride(shape: readonly number[]) {
 	return stride;
 }
 
-function to_c<D extends DataType>({ data, shape, stride }: Chunk<D>): Chunk<D> {
+function to_c<D extends zarr.DataType>({
+	data,
+	shape,
+	stride,
+}: zarr.Chunk<D>): zarr.Chunk<D> {
 	let size = shape.reduce((a, b) => a * b, 1);
 	// @ts-expect-error - We know constructor exists on TypedArray
 	let out = ndarray(new data.constructor(size), shape);

packages/core/__tests__/indexing/slice.test.ts

@@ -0,0 +1,130 @@
+import { describe, expect, it } from "vitest";
+
+import * as zarr from "../../src/index.js";
+import { get, set, slice } from "../../src/index.js";
+import { IndexError } from "../../src/indexing/indexer.js";
+
+const DATA = {
+	// biome-ignore format: the array should not be formatted
+	data: new Int32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]),
+	shape: [2, 3, 4],
+	stride: [12, 4, 1],
+};
+
+describe("slice", async () => {
+	let arr = await zarr.create(new Map(), {
+		shape: [2, 3, 4],
+		data_type: "int32",
+		chunk_shape: [1, 2, 2],
+	});
+	await set(arr, null, DATA);
+
+	it.each([
+		undefined,
+		null,
+		[null, null, null],
+		[slice(null), slice(null), slice(null)],
+		[null, slice(null), slice(null)],
+	])("Reads entire array: selection = %j", async (sel) => {
+		let chunk = await get(arr, sel);
+		expect(chunk).toStrictEqual(DATA);
+	});
+
+	it.each([
+		[
+			[1, slice(1, 3), null],
+			{
+				data: new Int32Array([16, 17, 18, 19, 20, 21, 22, 23]),
+				shape: [2, 4],
+				stride: [4, 1],
+			},
+		],
+		[
+			[null, slice(1, 3), slice(2)],
+			{
+				data: new Int32Array([4, 5, 8, 9, 16, 17, 20, 21]),
+				shape: [2, 2, 2],
+				stride: [4, 2, 1],
+			},
+		],
+		[
+			[null, null, 0],
+			{
+				data: new Int32Array([0, 4, 8, 12, 16, 20]),
+				shape: [2, 3],
+				stride: [3, 1],
+			},
+		],
+		[
+			[slice(3), slice(2), slice(2)],
+			{
+				data: new Int32Array([0, 1, 4, 5, 12, 13, 16, 17]),
+				shape: [2, 2, 2],
+				stride: [4, 2, 1],
+			},
+		],
+		[
+			[1, null, null],
+			{
+				// biome-ignore format: the array should not be formatted
+				data: new Int32Array([12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]),
+				shape: [3, 4],
+				stride: [4, 1],
+			},
+		],
+		[
+			[0, slice(null, null, 2), slice(null, null, 3)],
+			{
+				data: new Int32Array([0, 3, 8, 11]),
+				shape: [2, 2],
+				stride: [2, 1],
+			},
+		],
+		[
+			[null, null, slice(null, null, 4)],
+			{
+				data: new Int32Array([0, 4, 8, 12, 16, 20]),
+				shape: [2, 3, 1],
+				stride: [3, 1, 1],
+			},
+		],
+		[
+			[1, null, slice(null, 3, 2)],
+			{
+				data: new Int32Array([12, 14, 16, 18, 20, 22]),
+				shape: [3, 2],
+				stride: [2, 1],
+			},
+		],
+		[
+			[null, 1, null],
+			{
+				data: new Int32Array([4, 5, 6, 7, 16, 17, 18, 19]),
+				shape: [2, 4],
+				stride: [4, 1],
+			},
+		],
+		[
+			[1, 2, null],
+			{
+				data: new Int32Array([20, 21, 22, 23]),
+				shape: [4],
+				stride: [1],
+			},
+		],
+	])("Reads fancy slices: selection - %j", async (sel, expected) => {
+		let { data, shape, stride } = await get(arr, sel);
+		expect({ data, shape, stride }).toStrictEqual(expected);
+	});
+
+	it("Reads a scalar", async () => {
+		let sel = [1, 1, 1];
+		let val = await get(arr, sel);
+		expect(val).toBe(17);
+	});
+
+	it("Does not support negative indices", async () => {
+		let sel = [0, slice(null, null, -2), slice(null, null, 3)];
+		await expect(get(arr, sel)).rejects.toThrowError(IndexError);
+	});
+});

packages/indexing/__tests__/util.test.ts → packages/core/__tests__/indexing/util.test.ts

@@ -1,5 +1,6 @@
 import { describe, expect, test } from "vitest";
-import { range, slice, slice_indices } from "../src/util.js";
+
+import { range, slice, slice_indices } from "../../src/indexing/util.js";
 
 describe("slice", () => {
 	test("slice(null)", () => {

packages/core/package.json

@@ -17,6 +17,12 @@
 		"@zarrita/storage": "workspace:^",
 		"numcodecs": "^0.3.2"
 	},
+	"devDependencies": {
+		"@types/ndarray": "^1.0.11",
+		"@types/ndarray-ops": "^1.2.4",
+		"ndarray": "^1.0.19",
+		"ndarray-ops": "^1.2.2"
+	},
 	"publishConfig": {
 		"main": "dist/src/index.js",
 		"exports": {

packages/core/src/index.ts

@@ -4,15 +4,27 @@ export {
 	ByteStringArray,
 	UnicodeStringArray,
 } from "./typedarray.js";
-export {
-	Array,
-	get_context as _internal_get_array_context,
-	Group,
-	Location,
-	root,
-} from "./hierarchy.js";
+export { Array, Group, Location, root } from "./hierarchy.js";
 export { open } from "./open.js";
 export { create } from "./create.js";
 export { registry } from "./codecs.js";
-export * from "./consolidated.js";
+export { get, set } from "./indexing/ops.js";
+export { slice } from "./indexing/util.js";
+export { IndexError } from "./indexing/indexer.js";
+export { withConsolidated, tryWithConsolidated } from "./consolidated.js";
+
+export type { Listable } from "./consolidated.js";
 export type * from "./metadata.js";
+export type {
+	Slice,
+	Indices,
+	Projection,
+	GetOptions,
+	SetOptions,
+} from "./indexing/types.js";
+
+// internal exports for @zarrita/ndarray
+export { get as _zarrita_internal_get } from "./indexing/get.js";
+export { set as _zarrita_internal_set } from "./indexing/set.js";
+export { get_strides as _zarrita_internal_get_strides } from "./util.js";
+export { slice_indices as _zarrita_internal_slice_indices } from "./indexing/util.js";

packages/indexing/src/get.ts → packages/core/src/indexing/get.ts

@@ -1,15 +1,15 @@
-import type { Array, Chunk, DataType, Scalar, TypedArray } from "@zarrita/core";
 import type { Readable } from "@zarrita/storage";
+
+import { type Array, get_context } from "../hierarchy.js";
+import type { Chunk, DataType, Scalar, TypedArray } from "../metadata.js";
+import { BasicIndexer } from "./indexer.js";
 import type {
 	GetOptions,
 	Prepare,
 	SetFromChunk,
 	SetScalar,
 	Slice,
 } from "./types.js";
-
-import { _internal_get_array_context } from "@zarrita/core";
-import { BasicIndexer } from "./indexer.js";
 import { create_queue } from "./util.js";
 
 function unwrap<D extends DataType>(
@@ -36,7 +36,7 @@ export async function get<
 ): Promise<
 	null extends Sel[number] ? Arr : Slice extends Sel[number] ? Arr : Scalar<D>
 > {
-	let context = _internal_get_array_context(arr);
+	let context = get_context(arr);
 	let indexer = new BasicIndexer({
 		selection,
 		shape: arr.shape,