Skip to content

Commit 444c8bd

Browse files
achingbrainachingbrain
authored
feat: add offline option to all operations (#51)
Adds an `offline` option that means Helia won't go to the network when blocks are missing, instead throwing `ERR_NOT_FOUND`.
1 parent 60514b8 commit 444c8bd

14 files changed

+160
-10
lines changed

‎packages/unixfs/src/commands/chmod.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ export async function chmod (cid: CID, mode: number, blockstore: Blocks, options
104104
return updatePathCids(root.cid, resolved, blockstore, opts)
105105
}
106106

107-
const block = await blockstore.get(resolved.cid)
107+
const block = await blockstore.get(resolved.cid, options)
108108
let metadata: UnixFS
109109
let links: PBLink[] = []
110110

‎packages/unixfs/src/commands/touch.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ export async function touch (cid: CID, blockstore: Blocks, options: Partial<Touc
108108
return updatePathCids(root.cid, resolved, blockstore, opts)
109109
}
110110

111-
const block = await blockstore.get(resolved.cid)
111+
const block = await blockstore.get(resolved.cid, options)
112112
let metadata: UnixFS
113113
let links: PBLink[] = []
114114

‎packages/unixfs/src/commands/utils/add-link.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,12 @@ export async function addLink (parent: Directory, child: Required<PBLink>, block
5252

5353
const result = await addToDirectory(parent, child, blockstore, options)
5454

55-
if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes)) {
55+
if (await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes, options)) {
5656
log('converting directory to sharded directory')
5757

5858
const converted = await convertToShardedDirectory(result, blockstore)
5959
result.cid = converted.cid
60-
result.node = dagPB.decode(await blockstore.get(converted.cid))
60+
result.node = dagPB.decode(await blockstore.get(converted.cid, options))
6161
}
6262

6363
return result

‎packages/unixfs/src/commands/utils/is-over-shard-threshold.ts

+6-5
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,15 @@ import { UnixFS } from 'ipfs-unixfs'
33
import { CID_V0, CID_V1 } from './dir-sharded.js'
44
import type { Blocks } from '@helia/interface/blocks'
55
import type { PBNode } from '@ipld/dag-pb'
6+
import type { AbortOptions } from '@libp2p/interfaces'
67

78
/**
89
* Estimate node size only based on DAGLink name and CID byte lengths
910
* https://github.com/ipfs/go-unixfsnode/blob/37b47f1f917f1b2f54c207682f38886e49896ef9/data/builder/directory.go#L81-L96
1011
*
1112
* If the node is a hamt sharded directory the calculation is based on if it was a regular directory.
1213
*/
13-
export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, threshold: number): Promise<boolean> {
14+
export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, threshold: number, options: AbortOptions): Promise<boolean> {
1415
if (node.Data == null) {
1516
throw new Error('DagPB node had no data')
1617
}
@@ -21,7 +22,7 @@ export async function isOverShardThreshold (node: PBNode, blockstore: Blocks, th
2122
if (unixfs.type === 'directory') {
2223
size = estimateNodeSize(node)
2324
} else if (unixfs.type === 'hamt-sharded-directory') {
24-
size = await estimateShardSize(node, 0, threshold, blockstore)
25+
size = await estimateShardSize(node, 0, threshold, blockstore, options)
2526
} else {
2627
throw new Error('Can only estimate the size of directories or shards')
2728
}
@@ -42,7 +43,7 @@ function estimateNodeSize (node: PBNode): number {
4243
return size
4344
}
4445

45-
async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blocks): Promise<number> {
46+
async function estimateShardSize (node: PBNode, current: number, max: number, blockstore: Blocks, options: AbortOptions): Promise<number> {
4647
if (current > max) {
4748
return max
4849
}
@@ -67,10 +68,10 @@ async function estimateShardSize (node: PBNode, current: number, max: number, bl
6768
current += link.Hash.bytes.byteLength
6869

6970
if (link.Hash.code === dagPb.code) {
70-
const block = await blockstore.get(link.Hash)
71+
const block = await blockstore.get(link.Hash, options)
7172
const node = dagPb.decode(block)
7273

73-
current += await estimateShardSize(node, current, max, blockstore)
74+
current += await estimateShardSize(node, current, max, blockstore, options)
7475
}
7576
}
7677

‎packages/unixfs/src/commands/utils/remove-link.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ export async function removeLink (parent: Directory, name: string, blockstore: B
4141

4242
const result = await removeFromShardedDirectory(parent, name, blockstore, options)
4343

44-
if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes))) {
44+
if (!(await isOverShardThreshold(result.node, blockstore, options.shardSplitThresholdBytes, options))) {
4545
log('converting shard to flat directory %c', parent.cid)
4646

4747
return convertToFlatDirectory(result, blockstore, options)

‎packages/unixfs/src/index.ts

+48
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,12 @@ export interface CatOptions extends AbortOptions, ProgressOptions<GetEvents> {
8080
* An optional path to allow reading files inside directories
8181
*/
8282
path?: string
83+
84+
/**
85+
* If true, do not perform any network operations and throw if blocks are
86+
* missing from the local store. (default: false)
87+
*/
88+
offline?: boolean
8389
}
8490

8591
/**
@@ -102,6 +108,12 @@ export interface ChmodOptions extends AbortOptions, ProgressOptions<GetEvents |
102108
* smaller than this value will be regular UnixFS directories.
103109
*/
104110
shardSplitThresholdBytes: number
111+
112+
/**
113+
* If true, do not perform any network operations and throw if blocks are
114+
* missing from the local store. (default: false)
115+
*/
116+
offline?: boolean
105117
}
106118

107119
/**
@@ -118,6 +130,12 @@ export interface CpOptions extends AbortOptions, ProgressOptions<GetEvents | Put
118130
* smaller than this value will be regular UnixFS directories.
119131
*/
120132
shardSplitThresholdBytes: number
133+
134+
/**
135+
* If true, do not perform any network operations and throw if blocks are
136+
* missing from the local store. (default: false)
137+
*/
138+
offline?: boolean
121139
}
122140

123141
/**
@@ -139,6 +157,12 @@ export interface LsOptions extends AbortOptions, ProgressOptions<GetEvents> {
139157
* Stop reading the directory contents after this many directory entries
140158
*/
141159
length?: number
160+
161+
/**
162+
* If true, do not perform any network operations and throw if blocks are
163+
* missing from the local store. (default: false)
164+
*/
165+
offline?: boolean
142166
}
143167

144168
/**
@@ -171,6 +195,12 @@ export interface MkdirOptions extends AbortOptions, ProgressOptions<GetEvents |
171195
* smaller than this value will be regular UnixFS directories.
172196
*/
173197
shardSplitThresholdBytes: number
198+
199+
/**
200+
* If true, do not perform any network operations and throw if blocks are
201+
* missing from the local store. (default: false)
202+
*/
203+
offline?: boolean
174204
}
175205

176206
/**
@@ -182,6 +212,12 @@ export interface RmOptions extends AbortOptions, ProgressOptions<GetEvents | Put
182212
* smaller than this value will be regular UnixFS directories.
183213
*/
184214
shardSplitThresholdBytes: number
215+
216+
/**
217+
* If true, do not perform any network operations and throw if blocks are
218+
* missing from the local store. (default: false)
219+
*/
220+
offline?: boolean
185221
}
186222

187223
/**
@@ -192,6 +228,12 @@ export interface StatOptions extends AbortOptions, ProgressOptions<GetEvents> {
192228
* An optional path to allow statting paths inside directories
193229
*/
194230
path?: string
231+
232+
/**
233+
* If true, do not perform any network operations and throw if blocks are
234+
* missing from the local store. (default: false)
235+
*/
236+
offline?: boolean
195237
}
196238

197239
/**
@@ -275,6 +317,12 @@ export interface TouchOptions extends AbortOptions, ProgressOptions<GetEvents |
275317
* smaller than this value will be regular UnixFS directories.
276318
*/
277319
shardSplitThresholdBytes: number
320+
321+
/**
322+
* If true, do not perform any network operations and throw if blocks are
323+
* missing from the local store. (default: false)
324+
*/
325+
offline?: boolean
278326
}
279327

280328
/**

‎packages/unixfs/test/cat.spec.ts

+12
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,18 @@ describe('cat', () => {
6767
.with.property('code', 'ERR_NOT_A_FILE')
6868
})
6969

70+
it('refuses to read missing blocks', async () => {
71+
const cid = await fs.addBytes(smallFile)
72+
73+
await blockstore.delete(cid)
74+
expect(blockstore.has(cid)).to.be.false()
75+
76+
await expect(drain(fs.cat(cid, {
77+
offline: true
78+
}))).to.eventually.be.rejected
79+
.with.property('code', 'ERR_NOT_FOUND')
80+
})
81+
7082
it('reads file from inside a sharded directory', async () => {
7183
const dirCid = await createShardedDirectory(blockstore)
7284
const fileCid = await fs.addBytes(smallFile)

‎packages/unixfs/test/chmod.spec.ts

+12
Original file line numberDiff line numberDiff line change
@@ -89,4 +89,16 @@ describe('chmod', () => {
8989
expect(updatedMode).to.not.equal(originalMode)
9090
expect(updatedMode).to.equal(0o777)
9191
})
92+
93+
it('refuses to chmod missing blocks', async () => {
94+
const cid = await fs.addBytes(smallFile)
95+
96+
await blockstore.delete(cid)
97+
expect(blockstore.has(cid)).to.be.false()
98+
99+
await expect(fs.chmod(cid, 0o777, {
100+
offline: true
101+
})).to.eventually.be.rejected
102+
.with.property('code', 'ERR_NOT_FOUND')
103+
})
92104
})

‎packages/unixfs/test/cp.spec.ts

+13
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
1010
import { unixfs, type UnixFS } from '../src/index.js'
1111
import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
1212
import { createSubshardedDirectory } from './fixtures/create-subsharded-directory.js'
13+
import { smallFile } from './fixtures/files.js'
1314
import type { Blockstore } from 'interface-blockstore'
1415

1516
describe('cp', () => {
@@ -179,4 +180,16 @@ describe('cp', () => {
179180

180181
expect(finalDirCid).to.eql(containingDirCid, 'adding a file to the imported dir did not result in the same CID')
181182
})
183+
184+
it('refuses to copy missing blocks', async () => {
185+
const cid = await fs.addBytes(smallFile)
186+
187+
await blockstore.delete(cid)
188+
expect(blockstore.has(cid)).to.be.false()
189+
190+
await expect(fs.cp(cid, cid, 'file.txt', {
191+
offline: true
192+
})).to.eventually.be.rejected
193+
.with.property('code', 'ERR_NOT_FOUND')
194+
})
182195
})

‎packages/unixfs/test/ls.spec.ts

+14
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
import { expect } from 'aegir/chai'
44
import { MemoryBlockstore } from 'blockstore-core'
55
import all from 'it-all'
6+
import drain from 'it-drain'
67
import { unixfs, type UnixFS } from '../src/index.js'
78
import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
9+
import { smallFile } from './fixtures/files.js'
810
import type { Blockstore } from 'interface-blockstore'
911
import type { CID } from 'multiformats/cid'
1012

@@ -118,4 +120,16 @@ describe('ls', () => {
118120
expect(files.length).to.equal(1)
119121
expect(files.filter(file => file.name === fileName)).to.be.ok()
120122
})
123+
124+
it('refuses to list missing blocks', async () => {
125+
const cid = await fs.addBytes(smallFile)
126+
127+
await blockstore.delete(cid)
128+
expect(blockstore.has(cid)).to.be.false()
129+
130+
await expect(drain(fs.ls(cid, {
131+
offline: true
132+
}))).to.eventually.be.rejected
133+
.with.property('code', 'ERR_NOT_FOUND')
134+
})
121135
})

‎packages/unixfs/test/mkdir.spec.ts

+13
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { MemoryBlockstore } from 'blockstore-core'
55
import all from 'it-all'
66
import { unixfs, type UnixFS } from '../src/index.js'
77
import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
8+
import { smallFile } from './fixtures/files.js'
89
import type { Blockstore } from 'interface-blockstore'
910
import type { Mtime } from 'ipfs-unixfs'
1011
import type { CID } from 'multiformats/cid'
@@ -116,4 +117,16 @@ describe('mkdir', () => {
116117
path: dirName
117118
})).to.eventually.have.nested.property('unixfs.type', 'directory')
118119
})
120+
121+
it('refuses to mkdir with missing blocks', async () => {
122+
const cid = await fs.addBytes(smallFile)
123+
124+
await blockstore.delete(cid)
125+
expect(blockstore.has(cid)).to.be.false()
126+
127+
await expect(fs.mkdir(cid, 'dir', {
128+
offline: true
129+
})).to.eventually.be.rejected
130+
.with.property('code', 'ERR_NOT_FOUND')
131+
})
119132
})

‎packages/unixfs/test/rm.spec.ts

+12
Original file line numberDiff line numberDiff line change
@@ -217,4 +217,16 @@ describe('rm', () => {
217217

218218
expect(containingDirCid).to.eql(importerCid)
219219
})
220+
221+
it('refuses to rm missing blocks', async () => {
222+
const cid = await fs.addBytes(smallFile)
223+
224+
await blockstore.delete(cid)
225+
expect(blockstore.has(cid)).to.be.false()
226+
227+
await expect(fs.rm(cid, 'dir', {
228+
offline: true
229+
})).to.eventually.be.rejected
230+
.with.property('code', 'ERR_NOT_FOUND')
231+
})
220232
})

‎packages/unixfs/test/stat.spec.ts

+12
Original file line numberDiff line numberDiff line change
@@ -197,4 +197,16 @@ describe('stat', function () {
197197
expect(stats.type).to.equal('file')
198198
expect(stats.fileSize).to.equal(4n)
199199
})
200+
201+
it('refuses to stat missing blocks', async () => {
202+
const cid = await fs.addBytes(smallFile)
203+
204+
await blockstore.delete(cid)
205+
expect(blockstore.has(cid)).to.be.false()
206+
207+
await expect(fs.stat(cid, {
208+
offline: true
209+
})).to.eventually.be.rejected
210+
.with.property('code', 'ERR_NOT_FOUND')
211+
})
200212
})

‎packages/unixfs/test/touch.spec.ts

+13
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { MemoryBlockstore } from 'blockstore-core'
55
import delay from 'delay'
66
import { unixfs, type UnixFS } from '../src/index.js'
77
import { createShardedDirectory } from './fixtures/create-sharded-directory.js'
8+
import { smallFile } from './fixtures/files.js'
89
import type { Blockstore } from 'interface-blockstore'
910
import type { CID } from 'multiformats/cid'
1011

@@ -145,4 +146,16 @@ describe('.files.touch', () => {
145146
.that.satisfies((s: bigint) => s > seconds)
146147
}
147148
})
149+
150+
it('refuses to touch missing blocks', async () => {
151+
const cid = await fs.addBytes(smallFile)
152+
153+
await blockstore.delete(cid)
154+
expect(blockstore.has(cid)).to.be.false()
155+
156+
await expect(fs.touch(cid, {
157+
offline: true
158+
})).to.eventually.be.rejected
159+
.with.property('code', 'ERR_NOT_FOUND')
160+
})
148161
})

0 commit comments

Comments
 (0)