Use URL-based path resolution in HTTPStore

CHANGELOG.md

@@ -3,6 +3,8 @@
 ## Release 0.5.2
 **Date:** Unreleased
 
+* Use `URL`-based path resolution in `HTTPStore`. Allows for forwarding of `URL.searchParams` if specified.
+
 ## Release 0.5.1
 **Date:** 2021-07-19
 

src/storage/httpStore.ts

@@ -1,5 +1,5 @@
 import { ValidStoreType, AsyncStore } from './types';
-import { IS_NODE, joinUrlParts } from '../util';
+import { IS_NODE, resolveUrl } from '../util';
 import { KeyError, HTTPError } from '../errors';
 
 enum HTTPMethod {
@@ -15,17 +15,17 @@ interface HTTPStoreOptions {
     supportedMethods?: HTTPMethod[];
 }
 
-export class HTTPStore implements AsyncStore<ArrayBuffer> {
+export class HTTPStore<UrlRoot extends string | URL=string> implements AsyncStore<ArrayBuffer> {
     listDir?: undefined;
     rmDir?: undefined;
     getSize?: undefined;
     rename?: undefined;
 
-    public url: string;
+    public url: UrlRoot;
     public fetchOptions: RequestInit;
     private supportedMethods: Set<HTTPMethod>;
 
-    constructor(url: string, options: HTTPStoreOptions = {}) {
+    constructor(url: UrlRoot, options: HTTPStoreOptions = {}) {
         this.url = url;
         const { fetchOptions = {}, supportedMethods = DEFAULT_METHODS } = options;
         this.fetchOptions = fetchOptions;
@@ -37,7 +37,7 @@ export class HTTPStore implements AsyncStore<ArrayBuffer> {
     }
 
     async getItem(item: string, opts?: RequestInit) {
-        const url = joinUrlParts(this.url, item);
+        const url = resolveUrl(this.url, item);
         const value = await fetch(url, { ...this.fetchOptions, ...opts });
 
         if (value.status === 404) {
@@ -59,7 +59,7 @@ export class HTTPStore implements AsyncStore<ArrayBuffer> {
         if (!this.supportedMethods.has(HTTPMethod.PUT)) {
           throw new Error('HTTP PUT no a supported method for store.');
         }
-        const url = joinUrlParts(this.url, item);
+        const url = resolveUrl(this.url, item);
         if (typeof value === 'string') {
             value = new TextEncoder().encode(value).buffer;
         }
@@ -72,7 +72,7 @@ export class HTTPStore implements AsyncStore<ArrayBuffer> {
     }
 
     async containsItem(item: string): Promise<boolean> {
-        const url = joinUrlParts(this.url, item);
+        const url = resolveUrl(this.url, item);
         // Just check headers if HEAD method supported
         const method = this.supportedMethods.has(HTTPMethod.HEAD) ? HTTPMethod.HEAD : HTTPMethod.GET;
         const value = await fetch(url, { ...this.fetchOptions, method });

src/util.ts

@@ -40,6 +40,7 @@ export function normalizeStoragePath(path: string | String | null): string {
 
     // convert backslash to forward slash
     path = path.replace(/\\/g, "/");
+
     // ensure no leading slash
     while (path.length > 0 && path[0] === '/') {
         path = path.slice(1);
@@ -192,22 +193,18 @@ export function getStrides(shape: number[]): number[] {
     return strides;
 }
 
-/**
- * Preserves (double) slashes earlier in the path, so this works better
- * for URLs. From https://stackoverflow.com/a/46427607/4178400
- * @param args parts of a path or URL to join.
- */
-export function joinUrlParts(...args: string[]) {
-    return args.map((part, i) => {
-        if (i === 0) {
-          return part.trim().replace(/[\/]*$/g, '');
-        } else {
-          return part.trim().replace(/(^[\/]*|[\/]*$)/g, '');
-        }
-      }).filter(x=>x.length).join('/');
+export function resolveUrl(root: string | URL, path: string): string {
+    const base = typeof root === 'string' ? new URL(root) : root;
+    if (!base.pathname.endsWith('/')) {
+        // ensure trailing slash so that base is resolved as _directory_
+        base.pathname += '/';
+    }
+    const resolved = new URL(path, base);
+    // copy search params to new URL
+    resolved.search = base.search;
+    return resolved.href;
 }
 
-
 /**
  * Swaps byte order in-place for a given TypedArray.
  * Used to flip endian-ness when getting/setting chunks from/to zarr store.

test/util.test.ts

@@ -76,15 +76,32 @@ describe("ArrayEquals1D works", () => {
     });
 });
 
-describe("URL joining works", () => {
-    test.each([
+describe("URL resolution works", () => {
+    test.each<[[string | URL, string], string]>([
         [["https://example.com", "bla"], "https://example.com/bla"],
-        [["https://example.com/my-store", "arr.zarr"], "https://example.com/my-store/arr.zarr"],
-        [["https://example.com/", "arr.zarr"], "https://example.com/arr.zarr"],
-        [["https://example.com/", "", "arr.zarr"], "https://example.com/arr.zarr"],
-        // eslint-disable-next-line @typescript-eslint/ban-types
-    ])("joins parts as expected: output %s, expected %p", (parts: string[] | String, expected: string) => {
-        expect(util.joinUrlParts(...parts)).toEqual(expected);
+        [["https://example.com/my-store", "data.zarr"], "https://example.com/my-store/data.zarr"],
+        [["https://example.com/", "data.zarr"], "https://example.com/data.zarr"],
+        [["https://example.com/?hello=world", "data.zarr"], "https://example.com/data.zarr?hello=world"],
+        [["https://example.com?hello=world", "data.zarr"], "https://example.com/data.zarr?hello=world"],
+        [["https://example.com/data.zarr/nested/arr/", ".zarray"], "https://example.com/data.zarr/nested/arr/.zarray"],
+        [["https://example.com/data.zarr/nested/arr", ".zarray"], "https://example.com/data.zarr/nested/arr/.zarray"],
+        [["https://example.com/data.zarr/nested/group", "../.zgroup"], "https://example.com/data.zarr/nested/.zgroup"],
+        [[(() => {
+            const root = new URL("https://example.com/arr.zarr/my-store/");
+            root.searchParams.set("hello", "world");
+            root.searchParams.set("foo", "bar");
+            return root;
+        })(), ".zarray"], "https://example.com/arr.zarr/my-store/.zarray?hello=world&foo=bar"],
+        [[(() => {
+            const root = new URL("https://example.com/arr.zarr/my-store/");
+            root.username = "foo";
+            root.password = "bar";
+            root.searchParams.set("hello", "world");
+            root.searchParams.set("foo", "bar");
+            return root;
+        })(), ".zarray"], "https://foo:bar@example.com/arr.zarr/my-store/.zarray?hello=world&foo=bar"],
+    ])("joins parts as expected: output %s, expected %p", ([root, path]: [string | URL, string], expected: string) => {
+        expect(util.resolveUrl(root, path)).toEqual(expected);
     });
 });