feat: add offline option to all operations (#51)

Adds an `offline` option that means Helia won't go to the network
when blocks are missing, instead throwing `ERR_NOT_FOUND`.

Author: achingbrain

packages/unixfs/src/commands/chmod.ts

@@ -104,7 +104,7 @@ export async function chmod (cid: CID, mode: number, blockstore: Blocks, options
     return updatePathCids(root.cid, resolved, blockstore, opts)
   }
 
-  const block = await blockstore.get(resolved.cid)
+  const block = await blockstore.get(resolved.cid, options)
   let metadata: UnixFS
   let links: PBLink[] = []
 

packages/unixfs/src/commands/touch.ts

@@ -108,7 +108,7 @@ export async function touch (cid: CID, blockstore: Blocks, options: Partial<Touc
     return updatePathCids(root.cid, resolved, blockstore, opts)
   }
 
-  const block = await blockstore.get(resolved.cid)
+  const block = await blockstore.get(resolved.cid, options)
   let metadata: UnixFS
   let links: PBLink[] = []
 

packages/unixfs/src/commands/utils/add-link.ts

@@ -52,12 +52,12 @@ export async function addLink (parent: Directory, child: Required<PBLink>, block
 
   const result = await addToDirectory(parent, child, blockstore, options)
 
-  if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes)) {
+  if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes, options)) {
     log('converting directory to sharded directory')
 
     const converted = await convertToShardedDirectory(result, blockstore)
     result.cid = converted.cid
-    result.node = dagPB.decode(await blockstore.get(converted.cid))
+    result.node = dagPB.decode(await blockstore.get(converted.cid, options))
   }
 
   return result

packages/unixfs/src/commands/utils/is-over-shard-threshold.ts

@@ -3,14 +3,15 @@ import { UnixFS } from 'ipfs-unixfs'
 import { CID_V0, CID_V1 } from './dir-sharded.js'
 import type { Blocks } from '@helia/interface/blocks'
 import type { PBNode } from '@ipld/dag-pb'
+import type { AbortOptions } from '@libp2p/interfaces'
 
 /**
  * Estimate node size only based on DAGLink name and CID byte lengths
  * https://github.com/ipfs/go-unixfsnode/blob/37b47f1f917f1b2f54c207682f38886e49896ef9/data/builder/directory.go#L81-L96
  *
  * If the node is a hamt sharded directory the calculation is based on if it was a regular directory.
  */
-export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, threshold: number): Promise<boolean> {
+export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, threshold: number, options: AbortOptions): Promise<boolean> {
   if (node.Data == null) {
     throw new Error('DagPB node had no data')
   }
@@ -21,7 +22,7 @@ export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, th
   if (unixfs.type === 'directory') {
     size = estimateNodeSize(node)
   } else if (unixfs.type === 'hamt-sharded-directory') {
-    size = await estimateShardSize(node, 0, threshold, blockstore)
+    size = await estimateShardSize(node, 0, threshold, blockstore, options)
   } else {
     throw new Error('Can only estimate the size of directories or shards')
   }
@@ -42,7 +43,7 @@ function estimateNodeSize (node: PBNode): number {
   return size
 }
 
-async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blocks): Promise<number> {
+async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blocks, options: AbortOptions): Promise<number> {
   if (current > max) {
     return max
   }
@@ -67,10 +68,10 @@ async function estimateShardSize (node: PBNode, current: number, max: number, bl
     current += link.Hash.bytes.byteLength
 
     if (link.Hash.code === dagPb.code) {
-      const block = await blockstore.get(link.Hash)
+      const block = await blockstore.get(link.Hash, options)
       const node = dagPb.decode(block)
 
-      current += await estimateShardSize(node, current, max, blockstore)
+      current += await estimateShardSize(node, current, max, blockstore, options)
     }
   }
 

packages/unixfs/src/commands/utils/remove-link.ts

@@ -41,7 +41,7 @@ export async function removeLink (parent: Directory, name: string, blockstore: B
 
     const result = await removeFromShardedDirectory(parent, name, blockstore, options)
 
-    if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes))) {
+    if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes, options))) {
       log('converting shard to flat directory %c', parent.cid)
 
       return convertToFlatDirectory(result, blockstore, options)

packages/unixfs/src/index.ts

@@ -80,6 +80,12 @@ export interface CatOptions extends AbortOptions, ProgressOptions<GetEvents> {
    * An optional path to allow reading files inside directories
    */
   path?: string
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -102,6 +108,12 @@ export interface ChmodOptions extends AbortOptions, ProgressOptions<GetEvents |
    * smaller than this value will be regular UnixFS directories.
    */
   shardSplitThresholdBytes: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -118,6 +130,12 @@ export interface CpOptions extends AbortOptions, ProgressOptions<GetEvents | Put
    * smaller than this value will be regular UnixFS directories.
    */
   shardSplitThresholdBytes: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -139,6 +157,12 @@ export interface LsOptions extends AbortOptions, ProgressOptions<GetEvents> {
    * Stop reading the directory contents after this many directory entries
    */
   length?: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -171,6 +195,12 @@ export interface MkdirOptions extends AbortOptions, ProgressOptions<GetEvents |
    * smaller than this value will be regular UnixFS directories.
    */
   shardSplitThresholdBytes: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -182,6 +212,12 @@ export interface RmOptions extends AbortOptions, ProgressOptions<GetEvents | Put
    * smaller than this value will be regular UnixFS directories.
    */
   shardSplitThresholdBytes: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -192,6 +228,12 @@ export interface StatOptions extends AbortOptions, ProgressOptions<GetEvents> {
    * An optional path to allow statting paths inside directories
    */
   path?: string
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**
@@ -275,6 +317,12 @@ export interface TouchOptions extends AbortOptions, ProgressOptions<GetEvents |
    * smaller than this value will be regular UnixFS directories.
    */
   shardSplitThresholdBytes: number
+
+  /**
+   * If true, do not perform any network operations and throw if blocks are
+   * missing from the local store. (default: false)
+   */
+  offline?: boolean
 }
 
 /**

packages/unixfs/test/cat.spec.ts

@@ -67,6 +67,18 @@ describe('cat', () => {
       .with.property('code', 'ERR_NOT_A_FILE')
   })
 
+  it('refuses to read missing blocks', async () => {
+    const cid = await fs.addBytes(smallFile)
+
+    await blockstore.delete(cid)
+    expect(blockstore.has(cid)).to.be.false()
+
+    await expect(drain(fs.cat(cid, {
+      offline: true
+    }))).to.eventually.be.rejected
+      .with.property('code', 'ERR_NOT_FOUND')
+  })
+
   it('reads file from inside a sharded directory', async () => {
     const dirCid = await createShardedDirectory(blockstore)
     const fileCid = await fs.addBytes(smallFile)

packages/unixfs/test/chmod.spec.ts

@@ -89,4 +89,16 @@ describe('chmod', () => {
     expect(updatedMode).to.not.equal(originalMode)
     expect(updatedMode).to.equal(0o777)
   })
+
+  it('refuses to chmod missing blocks', async () => {
+    const cid = await fs.addBytes(smallFile)
+
+    await blockstore.delete(cid)
+    expect(blockstore.has(cid)).to.be.false()
+
+    await expect(fs.chmod(cid, 0o777, {
+      offline: true
+    })).to.eventually.be.rejected
+      .with.property('code', 'ERR_NOT_FOUND')
+  })
 })

packages/unixfs/test/cp.spec.ts

@@ -10,6 +10,7 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { unixfs, type UnixFS } from '../src/index.js'
 import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
 import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js'
+import { smallFile } from './fixtures/files.js'
 import type { Blockstore } from 'interface-blockstore'
 
 describe('cp', () => {
@@ -179,4 +180,16 @@ describe('cp', () => {
 
     expect(finalDirCid).to.eql(containingDirCid, 'adding a file to the imported dir did not result in the same CID')
   })
+
+  it('refuses to copy missing blocks', async () => {
+    const cid = await fs.addBytes(smallFile)
+
+    await blockstore.delete(cid)
+    expect(blockstore.has(cid)).to.be.false()
+
+    await expect(fs.cp(cid, cid, 'file.txt', {
+      offline: true
+    })).to.eventually.be.rejected
+      .with.property('code', 'ERR_NOT_FOUND')
+  })
 })

packages/unixfs/test/ls.spec.ts

@@ -3,8 +3,10 @@
 import { expect } from 'aegir/chai'
 import { MemoryBlockstore } from 'blockstore-core'
 import all from 'it-all'
+import drain from 'it-drain'
 import { unixfs, type UnixFS } from '../src/index.js'
 import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
+import { smallFile } from './fixtures/files.js'
 import type { Blockstore } from 'interface-blockstore'
 import type { CID } from 'multiformats/cid'
 
@@ -118,4 +120,16 @@ describe('ls', () => {
     expect(files.length).to.equal(1)
     expect(files.filter(file => file.name === fileName)).to.be.ok()
   })
+
+  it('refuses to list missing blocks', async () => {
+    const cid = await fs.addBytes(smallFile)
+
+    await blockstore.delete(cid)
+    expect(blockstore.has(cid)).to.be.false()
+
+    await expect(drain(fs.ls(cid, {
+      offline: true
+    }))).to.eventually.be.rejected
+      .with.property('code', 'ERR_NOT_FOUND')
+  })
 })

packages/unixfs/test/mkdir.spec.ts

@@ -5,6 +5,7 @@ import { MemoryBlockstore } from 'blockstore-core'
 import all from 'it-all'
 import { unixfs, type UnixFS } from '../src/index.js'
 import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
+import { smallFile } from './fixtures/files.js'
 import type { Blockstore } from 'interface-blockstore'
 import type { Mtime } from 'ipfs-unixfs'
 import type { CID } from 'multiformats/cid'
@@ -116,4 +117,16 @@ describe('mkdir', () => {
       path: dirName
     })).to.eventually.have.nested.property('unixfs.type', 'directory')
   })
+
+  it('refuses to mkdir with missing blocks', async () => {
+    const cid = await fs.addBytes(smallFile)
+
+    await blockstore.delete(cid)
+    expect(blockstore.has(cid)).to.be.false()
+
+    await expect(fs.mkdir(cid, 'dir', {
+      offline: true
+    })).to.eventually.be.rejected
+      .with.property('code', 'ERR_NOT_FOUND')
+  })
 })

packages/unixfs/test/rm.spec.ts

@@ -217,4 +217,16 @@ describe('rm', () => {
 
     expect(containingDirCid).to.eql(importerCid)
   })
+
+  it('refuses to rm missing blocks', async () => {
+    const cid = await fs.addBytes(smallFile)
+
+    await blockstore.delete(cid)
+    expect(blockstore.has(cid)).to.be.false()
+
+    await expect(fs.rm(cid, 'dir', {
+      offline: true
+    })).to.eventually.be.rejected
+      .with.property('code', 'ERR_NOT_FOUND')
+  })
 })

packages/unixfs/test/stat.spec.ts

@@ -197,4 +197,16 @@ describe('stat', function () {
     expect(stats.type).to.equal('file')
     expect(stats.fileSize).to.equal(4n)
   })
+
+  it('refuses to stat missing blocks', async () => {
+    const cid = await fs.addBytes(smallFile)
+
+    await blockstore.delete(cid)
+    expect(blockstore.has(cid)).to.be.false()
+
+    await expect(fs.stat(cid, {
+      offline: true
+    })).to.eventually.be.rejected
+      .with.property('code', 'ERR_NOT_FOUND')
+  })
 })

packages/unixfs/test/touch.spec.ts

@@ -5,6 +5,7 @@ import { MemoryBlockstore } from 'blockstore-core'
 import delay from 'delay'
 import { unixfs, type UnixFS } from '../src/index.js'
 import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
+import { smallFile } from './fixtures/files.js'
 import type { Blockstore } from 'interface-blockstore'
 import type { CID } from 'multiformats/cid'
 
@@ -145,4 +146,16 @@ describe('.files.touch', () => {
         .that.satisfies((s: bigint) => s > seconds)
     }
   })
+
+  it('refuses to touch missing blocks', async () => {
+    const cid = await fs.addBytes(smallFile)
+
+    await blockstore.delete(cid)
+    expect(blockstore.has(cid)).to.be.false()
+
+    await expect(fs.touch(cid, {
+      offline: true
+    })).to.eventually.be.rejected
+      .with.property('code', 'ERR_NOT_FOUND')
+  })
 })