Skip to content

Commit 6f8c15b

Browse files
authored
fix: use unixfs exporter to traverse DAGs (#455)
Reuse the existing exporter `walkPath` method to traverse a DAG as it is clever enough to not load unecessary blocks which is important when traversing HAMTs.
1 parent 1ee6a4a commit 6f8c15b

File tree

5 files changed

+54
-55
lines changed

5 files changed

+54
-55
lines changed

packages/interop/src/verified-fetch-unixfs-dir.spec.ts

+1-2
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,7 @@ describe('@helia/verified-fetch - unixfs directory', () => {
7575
})
7676
})
7777

78-
// TODO: find a smaller car file so the test doesn't timeout locally or flake on CI
79-
describe.skip('HAMT-sharded directory', () => {
78+
describe('HAMT-sharded directory', () => {
8079
before(async () => {
8180
// from https://github.com/ipfs/gateway-conformance/blob/193833b91f2e9b17daf45c84afaeeae61d9d7c7e/fixtures/trustless_gateway_car/single-layer-hamt-with-multi-block-files.car
8281
await loadFixtureDataCar(controller, 'bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i-single-layer-hamt-with-multi-block-files.car')

packages/unixfs/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,7 @@
169169
"ipfs-unixfs": "^11.1.3",
170170
"ipfs-unixfs-exporter": "^13.5.0",
171171
"ipfs-unixfs-importer": "^15.2.4",
172+
"it-all": "^3.0.4",
172173
"it-glob": "^2.0.6",
173174
"it-last": "^3.0.4",
174175
"it-pipe": "^3.0.1",
@@ -183,7 +184,6 @@
183184
"blockstore-core": "^4.4.0",
184185
"delay": "^6.0.0",
185186
"iso-url": "^1.2.1",
186-
"it-all": "^3.0.4",
187187
"it-drain": "^3.0.5",
188188
"it-first": "^3.0.4",
189189
"it-to-buffer": "^4.0.5",

packages/unixfs/src/commands/utils/resolve.ts

+9-48
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { logger } from '@libp2p/logger'
2-
import { exporter } from 'ipfs-unixfs-exporter'
3-
import { DoesNotExistError, InvalidParametersError } from '../../errors.js'
2+
import { walkPath } from 'ipfs-unixfs-exporter'
3+
import all from 'it-all'
4+
import { DoesNotExistError } from '../../errors.js'
45
import { addLink } from './add-link.js'
56
import { cidToDirectory } from './cid-to-directory.js'
67
import { cidToPBLink } from './cid-to-pblink.js'
@@ -37,57 +38,17 @@ export async function resolve (cid: CID, path: string | undefined, blockstore: B
3738
return { cid }
3839
}
3940

40-
log('resolve "%s" under %c', path, cid)
41-
42-
const parts = path.split('/').filter(Boolean)
43-
const segments: Segment[] = [{
44-
name: '',
45-
cid,
46-
size: 0n
47-
}]
48-
49-
for (let i = 0; i < parts.length; i++) {
50-
const part = parts[i]
51-
const result = await exporter(cid, blockstore, options)
52-
53-
log('resolving "%s"', part, result)
54-
55-
if (result.type === 'file') {
56-
if (i < parts.length - 1) {
57-
throw new InvalidParametersError('Path was invalid')
58-
}
59-
60-
cid = result.cid
61-
} else if (result.type === 'directory') {
62-
let dirCid: CID | undefined
63-
64-
for await (const entry of result.content()) {
65-
if (entry.name === part) {
66-
dirCid = entry.cid
67-
break
68-
}
69-
}
70-
71-
if (dirCid == null) {
72-
throw new DoesNotExistError('Could not find path in directory')
73-
}
74-
75-
cid = dirCid
76-
77-
segments.push({
78-
name: part,
79-
cid,
80-
size: result.size
81-
})
82-
} else {
83-
throw new InvalidParametersError('Could not resolve path')
84-
}
41+
const p = `/ipfs/${cid}${path == null ? '' : `/${path}`}`
42+
const segments = await all(walkPath(p, blockstore, options))
43+
44+
if (segments.length === 0) {
45+
throw new DoesNotExistError('Could not find path in directory')
8546
}
8647

8748
log('resolved %s to %c', path, cid)
8849

8950
return {
90-
cid,
51+
cid: segments[segments.length - 1].cid,
9152
path,
9253
segments
9354
}

packages/unixfs/test/cat.spec.ts

+39
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import { expect } from 'aegir/chai'
44
import { MemoryBlockstore } from 'blockstore-core'
5+
import all from 'it-all'
56
import drain from 'it-drain'
67
import toBuffer from 'it-to-buffer'
78
import { unixfs, type UnixFS } from '../src/index.js'
@@ -92,4 +93,42 @@ describe('cat', () => {
9293

9394
expect(bytes).to.deep.equal(smallFile)
9495
})
96+
97+
it('should only load blocks necessary to traverse a HAMT', async () => {
98+
const [, scriptFile, styleFile, imageFile, dir] = await all(fs.addAll([{
99+
path: 'index.html',
100+
content: Uint8Array.from([0, 1, 2])
101+
}, {
102+
path: 'script.js',
103+
content: Uint8Array.from([3, 4, 5])
104+
}, {
105+
path: 'style.css',
106+
content: Uint8Array.from([6, 7, 8])
107+
}, {
108+
path: 'image.png',
109+
content: Uint8Array.from([9, 0, 1])
110+
}], {
111+
shardSplitThresholdBytes: 1,
112+
wrapWithDirectory: true
113+
}))
114+
115+
const dirStat = await fs.stat(dir.cid)
116+
expect(dirStat.unixfs?.type).to.equal('hamt-sharded-directory')
117+
118+
// remove all blocks that aren't the index file
119+
await drain(blockstore.deleteMany([
120+
scriptFile.cid,
121+
styleFile.cid,
122+
imageFile.cid
123+
]))
124+
125+
// should be able to cat the index file without loading the other files
126+
// in the shard - the blockstore is offline so will throw if requested
127+
// blocks are not present
128+
const bytes = await toBuffer(fs.cat(dir.cid, {
129+
path: 'index.html'
130+
}))
131+
132+
expect(bytes).to.equalBytes(Uint8Array.from([0, 1, 2]))
133+
})
95134
})

packages/unixfs/test/rm.spec.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ describe('rm', () => {
3838
await expect(fs.stat(updatedDirCid, {
3939
path
4040
})).to.eventually.be.rejected
41-
.with.property('code', 'ERR_DOES_NOT_EXIST')
41+
.with.property('code', 'ERR_NOT_FOUND')
4242
})
4343

4444
it('removes a directory', async () => {
@@ -49,7 +49,7 @@ describe('rm', () => {
4949
await expect(fs.stat(updatedDirCid, {
5050
path
5151
})).to.eventually.be.rejected
52-
.with.property('code', 'ERR_DOES_NOT_EXIST')
52+
.with.property('code', 'ERR_NOT_FOUND')
5353
})
5454

5555
it('removes a sharded directory inside a normal directory', async () => {
@@ -67,7 +67,7 @@ describe('rm', () => {
6767
await expect(fs.stat(updatedContainingDirCid, {
6868
path: dirName
6969
})).to.eventually.be.rejected
70-
.with.property('code', 'ERR_DOES_NOT_EXIST')
70+
.with.property('code', 'ERR_NOT_FOUND')
7171
})
7272

7373
it('removes a sharded directory inside a sharded directory', async () => {
@@ -86,7 +86,7 @@ describe('rm', () => {
8686
await expect(fs.stat(updatedContainingDirCid, {
8787
path: dirName
8888
})).to.eventually.be.rejected
89-
.with.property('code', 'ERR_DOES_NOT_EXIST')
89+
.with.property('code', 'ERR_NOT_FOUND')
9090

9191
expect(updatedContainingDirCid.toString()).to.equal(shardedDirCid.toString(), 'adding and removing a file from a sharded directory did not result in the original sharded CID')
9292
})

0 commit comments

Comments
 (0)