fix: align defaults with filecoin (#6)

Filecoin uses a different set of defaults for importing files which
will be more performant for file transfers so use them.

The increased `maxChildrenPerNode` means fewer levels in a DAG so
the parallisation of DAG layer requests fetches more data and the
increased chunk size means fewer siblings in a layer.

Raw leaves removes some extra bytes from the leaf nodes.

Author: achingbrain

src/commands/add.ts

@@ -1,25 +1,53 @@
 import type { CID } from 'multiformats/cid'
-import type { Blockstore } from 'interface-blockstore'
+import type { Blockstore } from 'ipfs-unixfs-importer'
 import { ByteStream, DirectoryCandidate, FileCandidate, importBytes, importByteStream, ImportCandidateStream, importDirectory, importer, ImporterOptions, importFile, ImportResult } from 'ipfs-unixfs-importer'
+import { balanced } from 'ipfs-unixfs-importer/layout'
+import { fixedSize } from 'ipfs-unixfs-importer/chunker'
+
+/**
+ * Default importer settings match Filecoin
+ */
+const defaultImporterSettings: ImporterOptions = {
+  cidVersion: 1,
+  rawLeaves: true,
+  layout: balanced({
+    maxChildrenPerNode: 1024
+  }),
+  chunker: fixedSize({
+    chunkSize: 1048576
+  })
+}
 
 export async function * addAll (source: ImportCandidateStream, blockstore: Blockstore, options: Partial<ImporterOptions> = {}): AsyncGenerator<ImportResult, void, unknown> {
-  yield * importer(source, blockstore, options)
+  yield * importer(source, blockstore, {
+    ...defaultImporterSettings,
+    ...options
+  })
 }
 
 export async function addBytes (bytes: Uint8Array, blockstore: Blockstore, options: Partial<ImporterOptions> = {}): Promise<CID> {
-  const { cid } = await importBytes(bytes, blockstore, options)
+  const { cid } = await importBytes(bytes, blockstore, {
+    ...defaultImporterSettings,
+    ...options
+  })
 
   return cid
 }
 
 export async function addByteStream (bytes: ByteStream, blockstore: Blockstore, options: Partial<ImporterOptions> = {}): Promise<CID> {
-  const { cid } = await importByteStream(bytes, blockstore, options)
+  const { cid } = await importByteStream(bytes, blockstore, {
+    ...defaultImporterSettings,
+    ...options
+  })
 
   return cid
 }
 
 export async function addFile (file: FileCandidate, blockstore: Blockstore, options: Partial<ImporterOptions> = {}): Promise<CID> {
-  const { cid } = await importFile(file, blockstore, options)
+  const { cid } = await importFile(file, blockstore, {
+    ...defaultImporterSettings,
+    ...options
+  })
 
   return cid
 }
@@ -28,7 +56,10 @@ export async function addDirectory (dir: Partial<DirectoryCandidate>, blockstore
   const { cid } = await importDirectory({
     ...dir,
     path: dir.path ?? '-'
-  }, blockstore, options)
+  }, blockstore, {
+    ...defaultImporterSettings,
+    ...options
+  })
 
   return cid
 }

src/commands/utils/persist.ts

@@ -1,7 +1,7 @@
 import { CID } from 'multiformats/cid'
 import * as dagPb from '@ipld/dag-pb'
 import { sha256 } from 'multiformats/hashes/sha2'
-import type { Blockstore } from 'interface-blockstore'
+import type { Blockstore } from 'ipfs-unixfs-importer'
 import type { BlockCodec } from 'multiformats/codecs/interface'
 import type { Version as CIDVersion } from 'multiformats/cid'
 

test/fixtures/files.ts

@@ -1,4 +1,5 @@
+const ONE_MEG = 1024 * 1024
 
-export const largeFile = Uint8Array.from(new Array(490668).fill(0).map(() => Math.random() * 100))
+export const largeFile = Uint8Array.from(new Array(ONE_MEG * 5).fill(0).map(() => Math.random() * 100))
 
 export const smallFile = Uint8Array.from(new Array(13).fill(0).map(() => Math.random() * 100))

test/stat.spec.ts

@@ -38,23 +38,23 @@ describe('stat', function () {
     const block = await blockstore.get(largeFileCid)
     const node = dagPb.decode(block)
 
-    expect(node.Links).to.have.lengthOf(2)
+    expect(node.Links).to.have.lengthOf(5)
 
     await expect(fs.stat(largeFileCid)).to.eventually.include({
-      fileSize: 490668n,
-      blocks: 3,
-      localDagSize: 490776n
+      fileSize: 5242880n,
+      blocks: 6,
+      localDagSize: 5243139n
     })
 
     // remove one of the blocks so we now have an incomplete DAG
     await blockstore.delete(node.Links[0].Hash)
 
     // block count and local file/dag sizes should be smaller
     await expect(fs.stat(largeFileCid)).to.eventually.include({
-      fileSize: 490668n,
-      blocks: 2,
-      localFileSize: 228524n,
-      localDagSize: 228632n
+      fileSize: 5242880n,
+      blocks: 5,
+      localFileSize: 4194304n,
+      localDagSize: 4194563n
     })
   })
 
@@ -88,8 +88,8 @@ describe('stat', function () {
 
     await expect(fs.stat(cid)).to.eventually.include({
       fileSize: BigInt(largeFile.length),
-      dagSize: 490682n,
-      blocks: 3,
+      dagSize: 5242907n,
+      blocks: 6,
       type: 'file'
     })
   })