Skip to content

Commit a6bd198

Browse files
authored
fix: align defaults with filecoin (#6)
Filecoin uses a different set of defaults for importing files which will be more performant for file transfers so use them. The increased `maxChildrenPerNode` means fewer levels in a DAG so the parallisation of DAG layer requests fetches more data and the increased chunk size means fewer siblings in a layer. Raw leaves removes some extra bytes from the leaf nodes.
1 parent 4332c4c commit a6bd198

File tree

4 files changed

+50
-18
lines changed

4 files changed

+50
-18
lines changed

src/commands/add.ts

+37-6
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,53 @@
11
import type { CID } from 'multiformats/cid'
2-
import type { Blockstore } from 'interface-blockstore'
2+
import type { Blockstore } from 'ipfs-unixfs-importer'
33
import { ByteStream, DirectoryCandidate, FileCandidate, importBytes, importByteStream, ImportCandidateStream, importDirectory, importer, ImporterOptions, importFile, ImportResult } from 'ipfs-unixfs-importer'
4+
import { balanced } from 'ipfs-unixfs-importer/layout'
5+
import { fixedSize } from 'ipfs-unixfs-importer/chunker'
6+
7+
/**
8+
* Default importer settings match Filecoin
9+
*/
10+
const defaultImporterSettings: ImporterOptions = {
11+
cidVersion: 1,
12+
rawLeaves: true,
13+
layout: balanced({
14+
maxChildrenPerNode: 1024
15+
}),
16+
chunker: fixedSize({
17+
chunkSize: 1048576
18+
})
19+
}
420

521
export async function * addAll (source: ImportCandidateStream, blockstore: Blockstore, options: Partial<ImporterOptions> = {}): AsyncGenerator<ImportResult, void, unknown> {
6-
yield * importer(source, blockstore, options)
22+
yield * importer(source, blockstore, {
23+
...defaultImporterSettings,
24+
...options
25+
})
726
}
827

928
export async function addBytes (bytes: Uint8Array, blockstore: Blockstore, options: Partial<ImporterOptions> = {}): Promise<CID> {
10-
const { cid } = await importBytes(bytes, blockstore, options)
29+
const { cid } = await importBytes(bytes, blockstore, {
30+
...defaultImporterSettings,
31+
...options
32+
})
1133

1234
return cid
1335
}
1436

1537
export async function addByteStream (bytes: ByteStream, blockstore: Blockstore, options: Partial<ImporterOptions> = {}): Promise<CID> {
16-
const { cid } = await importByteStream(bytes, blockstore, options)
38+
const { cid } = await importByteStream(bytes, blockstore, {
39+
...defaultImporterSettings,
40+
...options
41+
})
1742

1843
return cid
1944
}
2045

2146
export async function addFile (file: FileCandidate, blockstore: Blockstore, options: Partial<ImporterOptions> = {}): Promise<CID> {
22-
const { cid } = await importFile(file, blockstore, options)
47+
const { cid } = await importFile(file, blockstore, {
48+
...defaultImporterSettings,
49+
...options
50+
})
2351

2452
return cid
2553
}
@@ -28,7 +56,10 @@ export async function addDirectory (dir: Partial<DirectoryCandidate>, blockstore
2856
const { cid } = await importDirectory({
2957
...dir,
3058
path: dir.path ?? '-'
31-
}, blockstore, options)
59+
}, blockstore, {
60+
...defaultImporterSettings,
61+
...options
62+
})
3263

3364
return cid
3465
}

src/commands/utils/persist.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { CID } from 'multiformats/cid'
22
import * as dagPb from '@ipld/dag-pb'
33
import { sha256 } from 'multiformats/hashes/sha2'
4-
import type { Blockstore } from 'interface-blockstore'
4+
import type { Blockstore } from 'ipfs-unixfs-importer'
55
import type { BlockCodec } from 'multiformats/codecs/interface'
66
import type { Version as CIDVersion } from 'multiformats/cid'
77

test/fixtures/files.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1+
const ONE_MEG = 1024 * 1024
12

2-
export const largeFile = Uint8Array.from(new Array(490668).fill(0).map(() => Math.random() * 100))
3+
export const largeFile = Uint8Array.from(new Array(ONE_MEG * 5).fill(0).map(() => Math.random() * 100))
34

45
export const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100))

test/stat.spec.ts

+10-10
Original file line numberDiff line numberDiff line change
@@ -38,23 +38,23 @@ describe('stat', function () {
3838
const block = await blockstore.get(largeFileCid)
3939
const node = dagPb.decode(block)
4040

41-
expect(node.Links).to.have.lengthOf(2)
41+
expect(node.Links).to.have.lengthOf(5)
4242

4343
await expect(fs.stat(largeFileCid)).to.eventually.include({
44-
fileSize: 490668n,
45-
blocks: 3,
46-
localDagSize: 490776n
44+
fileSize: 5242880n,
45+
blocks: 6,
46+
localDagSize: 5243139n
4747
})
4848

4949
// remove one of the blocks so we now have an incomplete DAG
5050
await blockstore.delete(node.Links[0].Hash)
5151

5252
// block count and local file/dag sizes should be smaller
5353
await expect(fs.stat(largeFileCid)).to.eventually.include({
54-
fileSize: 490668n,
55-
blocks: 2,
56-
localFileSize: 228524n,
57-
localDagSize: 228632n
54+
fileSize: 5242880n,
55+
blocks: 5,
56+
localFileSize: 4194304n,
57+
localDagSize: 4194563n
5858
})
5959
})
6060

@@ -88,8 +88,8 @@ describe('stat', function () {
8888

8989
await expect(fs.stat(cid)).to.eventually.include({
9090
fileSize: BigInt(largeFile.length),
91-
dagSize: 490682n,
92-
blocks: 3,
91+
dagSize: 5242907n,
92+
blocks: 6,
9393
type: 'file'
9494
})
9595
})

0 commit comments

Comments
 (0)