Skip to content

Commit de78f4d

Browse files
authored
feat: expose progress events from importer, blockstore and bitswap (#13)
Expose progress event types that are passed to `onProgress` callback passed to all operations.
1 parent 4c8d124 commit de78f4d

File tree

6 files changed

+127
-31
lines changed

6 files changed

+127
-31
lines changed

packages/interop/package.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,9 @@
6464
"go-ipfs": "^0.18.1",
6565
"helia": "next",
6666
"ipfs-core-types": "^0.14.0",
67-
"ipfs-unixfs-importer": "^15.0.0",
67+
"ipfs-unixfs-importer": "^15.0.1",
6868
"ipfsd-ctl": "^13.0.0",
69+
"it-to-buffer": "^3.0.1",
6970
"kubo-rpc-client": "^3.0.0",
7071
"libp2p": "next",
7172
"merge-options": "^3.0.4",

packages/interop/test/bitswap.spec.ts

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/* eslint-env mocha */
2+
3+
import { expect } from 'aegir/chai'
4+
import { createHeliaNode } from './fixtures/create-helia.js'
5+
import { createKuboNode } from './fixtures/create-kubo.js'
6+
import type { Helia } from '@helia/interface'
7+
import type { Controller } from 'ipfsd-ctl'
8+
import { UnixFS, unixfs } from '@helia/unixfs'
9+
import type { FileCandidate } from 'ipfs-unixfs-importer'
10+
import toBuffer from 'it-to-buffer'
11+
12+
describe('unixfs bitswap interop', () => {
13+
let helia: Helia
14+
let unixFs: UnixFS
15+
let kubo: Controller
16+
17+
beforeEach(async () => {
18+
helia = await createHeliaNode()
19+
unixFs = unixfs(helia)
20+
kubo = await createKuboNode()
21+
22+
// connect helia to kubo
23+
await helia.libp2p.peerStore.addressBook.add(kubo.peer.id, kubo.peer.addresses)
24+
await helia.libp2p.dial(kubo.peer.id)
25+
})
26+
27+
afterEach(async () => {
28+
if (helia != null) {
29+
await helia.stop()
30+
}
31+
32+
if (kubo != null) {
33+
await kubo.stop()
34+
}
35+
})
36+
37+
it('should add a large file to helia and fetch it from kubo', async () => {
38+
const chunkSize = 1024 * 1024
39+
const size = chunkSize * 10
40+
const input: Uint8Array[] = []
41+
42+
const candidate: FileCandidate = {
43+
content: (async function * () {
44+
for (let i = 0; i < size; i += chunkSize) {
45+
const buf = new Uint8Array(chunkSize)
46+
input.push(buf)
47+
48+
yield buf
49+
}
50+
}())
51+
}
52+
53+
const cid = await unixFs.addFile(candidate)
54+
55+
const bytes = await toBuffer(kubo.api.cat(cid))
56+
57+
expect(bytes).to.equalBytes(await toBuffer(input))
58+
})
59+
60+
it('should add a large file to kubo and fetch it from helia', async () => {
61+
const chunkSize = 1024 * 1024
62+
const size = chunkSize * 10
63+
const input: Uint8Array[] = []
64+
65+
const candidate: FileCandidate = {
66+
content: (async function * () {
67+
for (let i = 0; i < size; i += chunkSize) {
68+
const buf = new Uint8Array(chunkSize)
69+
input.push(buf)
70+
71+
yield buf
72+
}
73+
}())
74+
}
75+
76+
const { cid } = await kubo.api.add(candidate.content)
77+
78+
const bytes = await toBuffer(unixFs.cat(cid))
79+
80+
expect(bytes).to.equalBytes(await toBuffer(input))
81+
})
82+
})

packages/interop/test/files.spec.ts

+6-6
Original file line numberDiff line numberDiff line change
@@ -5,31 +5,31 @@ import { createHeliaNode } from './fixtures/create-helia.js'
55
import { createKuboNode } from './fixtures/create-kubo.js'
66
import type { Helia } from '@helia/interface'
77
import type { Controller } from 'ipfsd-ctl'
8-
import { UnixFS, unixfs } from '@helia/unixfs'
8+
import { AddOptions, UnixFS, unixfs } from '@helia/unixfs'
99
import { balanced } from 'ipfs-unixfs-importer/layout'
1010
import { fixedSize } from 'ipfs-unixfs-importer/chunker'
11-
import type { FileCandidate, ImporterOptions } from 'ipfs-unixfs-importer'
11+
import type { FileCandidate } from 'ipfs-unixfs-importer'
1212
import type { CID } from 'multiformats/cid'
13-
import type { AddOptions } from 'ipfs-core-types/src/root.js'
13+
import type { AddOptions as KuboAddOptions } from 'ipfs-core-types/src/root.js'
1414

1515
describe('unixfs interop', () => {
1616
let helia: Helia
1717
let unixFs: UnixFS
1818
let kubo: Controller
1919

20-
async function importToHelia (data: FileCandidate, opts?: Partial<ImporterOptions>): Promise<CID> {
20+
async function importToHelia (data: FileCandidate, opts?: Partial<AddOptions>): Promise<CID> {
2121
const cid = await unixFs.addFile(data, opts)
2222

2323
return cid
2424
}
2525

26-
async function importToKubo (data: FileCandidate, opts?: AddOptions): Promise<CID> {
26+
async function importToKubo (data: FileCandidate, opts?: KuboAddOptions): Promise<CID> {
2727
const result = await kubo.api.add(data.content, opts)
2828

2929
return result.cid
3030
}
3131

32-
async function expectSameCid (data: () => FileCandidate, heliaOpts: Partial<ImporterOptions> = {}, kuboOpts: AddOptions = {}): Promise<void> {
32+
async function expectSameCid (data: () => FileCandidate, heliaOpts: Partial<AddOptions> = {}, kuboOpts: KuboAddOptions = {}): Promise<void> {
3333
const heliaCid = await importToHelia(data(), {
3434
// these are the default kubo options
3535
cidVersion: 0,

packages/unixfs/package.json

+6-2
Original file line numberDiff line numberDiff line change
@@ -147,12 +147,13 @@
147147
"hamt-sharding": "^3.0.2",
148148
"interface-blockstore": "^5.0.0",
149149
"ipfs-unixfs": "^11.0.0",
150-
"ipfs-unixfs-exporter": "^13.0.0",
151-
"ipfs-unixfs-importer": "^15.0.0",
150+
"ipfs-unixfs-exporter": "^13.0.1",
151+
"ipfs-unixfs-importer": "^15.0.1",
152152
"it-last": "^2.0.0",
153153
"it-pipe": "^2.0.5",
154154
"merge-options": "^3.0.4",
155155
"multiformats": "^11.0.1",
156+
"progress-events": "^1.0.0",
156157
"sparse-array": "^1.3.2"
157158
},
158159
"devDependencies": {
@@ -164,5 +165,8 @@
164165
"it-first": "^2.0.0",
165166
"it-to-buffer": "^3.0.0",
166167
"uint8arrays": "^4.0.3"
168+
},
169+
"typedoc": {
170+
"entryPoint": "./src/index.ts"
167171
}
168172
}

packages/unixfs/src/commands/chmod.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ export async function chmod (cid: CID, mode: number, blockstore: Blocks, options
3636
// but do not reimport files, only manipulate dag-pb nodes
3737
const root = await pipe(
3838
async function * () {
39-
for await (const entry of recursive(resolved.cid, blockstore)) {
39+
for await (const entry of recursive(resolved.cid, blockstore, options)) {
4040
let metadata: UnixFS
4141
let links: PBLink[] = []
4242

@@ -63,6 +63,7 @@ export async function chmod (cid: CID, mode: number, blockstore: Blocks, options
6363
}
6464
}
6565
},
66+
// @ts-expect-error cannot combine progress types
6667
(source) => importer(source, blockstore, {
6768
...opts,
6869
dagBuilder: async function * (source, block) {

packages/unixfs/src/index.ts

+29-21
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
*/
3333

3434
import type { CID, Version } from 'multiformats/cid'
35-
import type { Blocks } from '@helia/interface/blocks'
35+
import type { Blocks, GetBlockProgressEvents, PutBlockProgressEvents } from '@helia/interface/blocks'
3636
import type { AbortOptions } from '@libp2p/interfaces'
3737
import { addAll, addBytes, addByteStream, addDirectory, addFile } from './commands/add.js'
3838
import { cat } from './commands/cat.js'
@@ -45,16 +45,24 @@ import { touch } from './commands/touch.js'
4545
import { chmod } from './commands/chmod.js'
4646
import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
4747
import { ls } from './commands/ls.js'
48-
import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImportResult } from 'ipfs-unixfs-importer'
48+
import type { ByteStream, DirectoryCandidate, FileCandidate, ImportCandidateStream, ImporterOptions, ImportProgressEvents, ImportResult } from 'ipfs-unixfs-importer'
49+
import type { ProgressOptions } from 'progress-events'
4950

5051
export interface UnixFSComponents {
5152
blockstore: Blocks
5253
}
5354

55+
export type AddEvents = PutBlockProgressEvents
56+
| ImportProgressEvents
57+
58+
export interface AddOptions extends AbortOptions, Omit<ImporterOptions, 'onProgress'>, ProgressOptions<AddEvents> {
59+
60+
}
61+
5462
/**
5563
* Options to pass to the cat command
5664
*/
57-
export interface CatOptions extends AbortOptions {
65+
export interface CatOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents> {
5866
/**
5967
* Start reading the file at this offset
6068
*/
@@ -74,7 +82,7 @@ export interface CatOptions extends AbortOptions {
7482
/**
7583
* Options to pass to the chmod command
7684
*/
77-
export interface ChmodOptions extends AbortOptions {
85+
export interface ChmodOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents | PutBlockProgressEvents> {
7886
/**
7987
* If the target of the operation is a directory and this is true,
8088
* apply the new mode to all directory contents
@@ -96,7 +104,7 @@ export interface ChmodOptions extends AbortOptions {
96104
/**
97105
* Options to pass to the cp command
98106
*/
99-
export interface CpOptions extends AbortOptions {
107+
export interface CpOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents | PutBlockProgressEvents> {
100108
/**
101109
* If true, allow overwriting existing directory entries (default: false)
102110
*/
@@ -112,7 +120,7 @@ export interface CpOptions extends AbortOptions {
112120
/**
113121
* Options to pass to the ls command
114122
*/
115-
export interface LsOptions extends AbortOptions {
123+
export interface LsOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents> {
116124
/**
117125
* Optional path to list subdirectory contents if the target CID resolves to
118126
* a directory
@@ -133,7 +141,7 @@ export interface LsOptions extends AbortOptions {
133141
/**
134142
* Options to pass to the mkdir command
135143
*/
136-
export interface MkdirOptions extends AbortOptions {
144+
export interface MkdirOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents | PutBlockProgressEvents> {
137145
/**
138146
* The CID version to create the new directory with - defaults to the same
139147
* version as the containing directory
@@ -165,7 +173,7 @@ export interface MkdirOptions extends AbortOptions {
165173
/**
166174
* Options to pass to the rm command
167175
*/
168-
export interface RmOptions extends AbortOptions {
176+
export interface RmOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents | PutBlockProgressEvents> {
169177
/**
170178
* DAGs with a root block larger than this value will be sharded. Blocks
171179
* smaller than this value will be regular UnixFS directories.
@@ -176,7 +184,7 @@ export interface RmOptions extends AbortOptions {
176184
/**
177185
* Options to pass to the stat command
178186
*/
179-
export interface StatOptions extends AbortOptions {
187+
export interface StatOptions extends AbortOptions, ProgressOptions<GetBlockProgressEvents> {
180188
/**
181189
* An optional path to allow statting paths inside directories
182190
*/
@@ -292,7 +300,7 @@ export interface UnixFS {
292300
* }
293301
* ```
294302
*/
295-
addAll: (source: ImportCandidateStream, options?: Partial<ImporterOptions>) => AsyncIterable<ImportResult>
303+
addAll: (source: ImportCandidateStream, options?: Partial<AddOptions>) => AsyncIterable<ImportResult>
296304

297305
/**
298306
* Add a single `Uint8Array` to your Helia node as a file.
@@ -305,7 +313,7 @@ export interface UnixFS {
305313
* console.info(cid)
306314
* ```
307315
*/
308-
addBytes: (bytes: Uint8Array, options?: Partial<ImporterOptions>) => Promise<CID>
316+
addBytes: (bytes: Uint8Array, options?: Partial<AddOptions>) => Promise<CID>
309317

310318
/**
311319
* Add a stream of `Uint8Array` to your Helia node as a file.
@@ -321,7 +329,7 @@ export interface UnixFS {
321329
* console.info(cid)
322330
* ```
323331
*/
324-
addByteStream: (bytes: ByteStream, options?: Partial<ImporterOptions>) => Promise<CID>
332+
addByteStream: (bytes: ByteStream, options?: Partial<AddOptions>) => Promise<CID>
325333

326334
/**
327335
* Add a file to your Helia node with optional metadata.
@@ -342,7 +350,7 @@ export interface UnixFS {
342350
* console.info(cid)
343351
* ```
344352
*/
345-
addFile: (file: FileCandidate, options?: Partial<ImporterOptions>) => Promise<CID>
353+
addFile: (file: FileCandidate, options?: Partial<AddOptions>) => Promise<CID>
346354

347355
/**
348356
* Add a directory to your Helia node.
@@ -355,7 +363,7 @@ export interface UnixFS {
355363
* console.info(cid)
356364
* ```
357365
*/
358-
addDirectory: (dir?: Partial<DirectoryCandidate>, options?: Partial<ImporterOptions>) => Promise<CID>
366+
addDirectory: (dir?: Partial<DirectoryCandidate>, options?: Partial<AddOptions>) => Promise<CID>
359367

360368
/**
361369
* Retrieve the contents of a file from your Helia node.
@@ -368,7 +376,7 @@ export interface UnixFS {
368376
* }
369377
* ```
370378
*/
371-
cat: (cid: CID, options?: Partial<CatOptions>) => AsyncIterable<Uint8Array>
379+
cat: (cid: CID, options?: Partial<CatOptions> & ProgressOptions<GetBlockProgressEvents>) => AsyncIterable<Uint8Array>
372380

373381
/**
374382
* Change the permissions on a file or directory in a DAG
@@ -415,7 +423,7 @@ export interface UnixFS {
415423
* }
416424
* ```
417425
*/
418-
ls: (cid: CID, options?: Partial<LsOptions>) => AsyncIterable<UnixFSEntry>
426+
ls: (cid: CID, options?: Partial<LsOptions> & ProgressOptions<GetBlockProgressEvents>) => AsyncIterable<UnixFSEntry>
419427

420428
/**
421429
* Make a new directory under an existing directory.
@@ -489,23 +497,23 @@ class DefaultUnixFS implements UnixFS {
489497
this.components = components
490498
}
491499

492-
async * addAll (source: ImportCandidateStream, options: Partial<ImporterOptions> = {}): AsyncIterable<ImportResult> {
500+
async * addAll (source: ImportCandidateStream, options: Partial<AddOptions> = {}): AsyncIterable<ImportResult> {
493501
yield * addAll(source, this.components.blockstore, options)
494502
}
495503

496-
async addBytes (bytes: Uint8Array, options: Partial<ImporterOptions> = {}): Promise<CID> {
504+
async addBytes (bytes: Uint8Array, options: Partial<AddOptions> = {}): Promise<CID> {
497505
return await addBytes(bytes, this.components.blockstore, options)
498506
}
499507

500-
async addByteStream (bytes: ByteStream, options: Partial<ImporterOptions> = {}): Promise<CID> {
508+
async addByteStream (bytes: ByteStream, options: Partial<AddOptions> = {}): Promise<CID> {
501509
return await addByteStream(bytes, this.components.blockstore, options)
502510
}
503511

504-
async addFile (file: FileCandidate, options: Partial<ImporterOptions> = {}): Promise<CID> {
512+
async addFile (file: FileCandidate, options: Partial<AddOptions> = {}): Promise<CID> {
505513
return await addFile(file, this.components.blockstore, options)
506514
}
507515

508-
async addDirectory (dir: Partial<DirectoryCandidate> = {}, options: Partial<ImporterOptions> = {}): Promise<CID> {
516+
async addDirectory (dir: Partial<DirectoryCandidate> = {}, options: Partial<AddOptions> = {}): Promise<CID> {
509517
return await addDirectory(dir, this.components.blockstore, options)
510518
}
511519

0 commit comments

Comments
 (0)