Skip to content

Commit b490a6e

Browse files
authored
feat: add globSource and urlSource (#53)
Ports `globSource` and `urlSource` from `ipfs-utils` for use with `unixfs.add` and `unixfs.addAll`.
1 parent 26b5cd3 commit b490a6e

File tree

16 files changed

+508
-2
lines changed

16 files changed

+508
-2
lines changed

packages/unixfs/.aegir.js

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import EchoServer from 'aegir/echo-server'
2+
import { format } from 'iso-url'
3+
4+
export default {
5+
test: {
6+
async before (options) {
7+
let echoServer = new EchoServer()
8+
await echoServer.start()
9+
const { address, port } = echoServer.server.address()
10+
let hostname = address
11+
if(options.runner === 'react-native-android') {
12+
hostname = '10.0.2.2'
13+
}
14+
return {
15+
echoServer,
16+
env: { ECHO_SERVER : format({ protocol: 'http:', hostname, port })}
17+
}
18+
},
19+
async after (options, before) {
20+
await before.echoServer.stop()
21+
}
22+
}
23+
}

packages/unixfs/package.json

+11-1
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,7 @@
165165
"ipfs-unixfs": "^11.0.0",
166166
"ipfs-unixfs-exporter": "^13.1.0",
167167
"ipfs-unixfs-importer": "^15.1.0",
168+
"it-glob": "^2.0.4",
168169
"it-last": "^3.0.1",
169170
"it-pipe": "^3.0.1",
170171
"merge-options": "^3.0.4",
@@ -176,13 +177,22 @@
176177
"aegir": "^39.0.8",
177178
"blockstore-core": "^4.0.1",
178179
"delay": "^6.0.0",
180+
"iso-url": "^1.2.1",
179181
"it-all": "^3.0.1",
180182
"it-drain": "^3.0.1",
181183
"it-first": "^3.0.1",
182184
"it-to-buffer": "^4.0.1",
183-
"uint8arrays": "^4.0.3"
185+
"uint8arrays": "^4.0.3",
186+
"wherearewe": "^2.0.1"
184187
},
185188
"typedoc": {
186189
"entryPoint": "./src/index.ts"
190+
},
191+
"browser": {
192+
"./dist/utils/glob-source.js": false,
193+
"node:fs": false,
194+
"node:fs/promises": false,
195+
"node:path": false,
196+
"node:url": false
187197
}
188198
}

packages/unixfs/src/index.ts

+15
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,18 @@
2929
* console.info(entry)
3030
* }
3131
* ```
32+
*
33+
* @example
34+
*
35+
* Recursively adding a directory (Node.js-compatibly environments only):
36+
*
37+
* ```typescript
38+
* import { globSource } from '@helia/unixfs'
39+
*
40+
* for await (const entry of fs.addAll(globSource('path/to/containing/dir', 'glob-pattern'))) {
41+
* console.info(entry)
42+
* }
43+
* ```
3244
*/
3345

3446
import { addAll, addBytes, addByteStream, addDirectory, addFile } from './commands/add.js'
@@ -607,3 +619,6 @@ class DefaultUnixFS implements UnixFS {
607619
export function unixfs (helia: { blockstore: Blocks }): UnixFS {
608620
return new DefaultUnixFS(helia)
609621
}
622+
623+
export { globSource } from './utils/glob-source.js'
624+
export { urlSource } from './utils/url-source.js'
+92
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import fs from 'node:fs'
2+
import fsp from 'node:fs/promises'
3+
import Path from 'node:path'
4+
import glob from 'it-glob'
5+
import { InvalidParametersError } from '../errors.js'
6+
import type { MtimeLike } from 'ipfs-unixfs'
7+
import type { ImportCandidateStream } from 'ipfs-unixfs-importer'
8+
9+
export interface GlobSourceOptions {
10+
/**
11+
* Include .dot files in matched paths
12+
*/
13+
hidden?: boolean
14+
15+
/**
16+
* follow symlinks
17+
*/
18+
followSymlinks?: boolean
19+
20+
/**
21+
* Preserve mode
22+
*/
23+
preserveMode?: boolean
24+
25+
/**
26+
* Preserve mtime
27+
*/
28+
preserveMtime?: boolean
29+
30+
/**
31+
* mode to use - if preserveMode is true this will be ignored
32+
*/
33+
mode?: number
34+
35+
/**
36+
* mtime to use - if preserveMtime is true this will be ignored
37+
*/
38+
mtime?: MtimeLike
39+
}
40+
41+
export interface GlobSourceResult {
42+
path: string
43+
content: AsyncIterable<Uint8Array> | undefined
44+
mode: number | undefined
45+
mtime: MtimeLike | undefined
46+
}
47+
48+
/**
49+
* Create an async iterator that yields paths that match requested glob pattern
50+
*/
51+
export async function * globSource (cwd: string, pattern: string, options: GlobSourceOptions = {}): ImportCandidateStream {
52+
if (typeof pattern !== 'string') {
53+
throw new InvalidParametersError('Pattern must be a string')
54+
}
55+
56+
if (!Path.isAbsolute(cwd)) {
57+
cwd = Path.resolve(process.cwd(), cwd)
58+
}
59+
60+
const globOptions = Object.assign({}, {
61+
nodir: false,
62+
realpath: false,
63+
absolute: true,
64+
dot: Boolean(options.hidden),
65+
follow: options.followSymlinks != null ? options.followSymlinks : true
66+
})
67+
68+
for await (const p of glob(cwd, pattern, globOptions)) {
69+
const stat = await fsp.stat(p)
70+
71+
let mode = options.mode
72+
73+
if (options.preserveMode === true) {
74+
mode = stat.mode
75+
}
76+
77+
let mtime = options.mtime
78+
79+
if (options.preserveMtime === true) {
80+
mtime = stat.mtime
81+
}
82+
83+
yield {
84+
path: toPosix(p.replace(cwd, '')),
85+
content: stat.isFile() ? fs.createReadStream(p) : undefined,
86+
mode,
87+
mtime
88+
}
89+
}
90+
}
91+
92+
const toPosix = (path: string): string => path.replace(/\\/g, '/')
+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
import { UnknownError } from '../errors.js'
2+
import type { FileCandidate } from 'ipfs-unixfs-importer'
3+
4+
export function urlSource (url: URL, options?: RequestInit): FileCandidate<AsyncGenerator<Uint8Array, void, unknown>> {
5+
return {
6+
path: decodeURIComponent(new URL(url).pathname.split('/').pop() ?? ''),
7+
content: readURLContent(url, options)
8+
}
9+
}
10+
11+
async function * readURLContent (url: URL, options?: RequestInit): AsyncGenerator<Uint8Array, void, unknown> {
12+
const response = await globalThis.fetch(url, options)
13+
14+
if (response.body == null) {
15+
throw new UnknownError('HTTP response did not have a body')
16+
}
17+
18+
const reader = response.body.getReader()
19+
20+
try {
21+
while (true) {
22+
const { done, value } = await reader.read()
23+
24+
if (done) {
25+
return
26+
}
27+
28+
if (value != null) {
29+
yield value
30+
}
31+
}
32+
} finally {
33+
reader.releaseLock()
34+
}
35+
}

packages/unixfs/test/add.spec.ts

+19-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@
33
import { expect } from 'aegir/chai'
44
import { MemoryBlockstore } from 'blockstore-core'
55
import all from 'it-all'
6-
import { unixfs, type UnixFS } from '../src/index.js'
6+
import last from 'it-last'
7+
import { isNode } from 'wherearewe'
8+
import { globSource, unixfs, urlSource, type UnixFS } from '../src/index.js'
79
import type { Blockstore } from 'interface-blockstore'
810

911
describe('addAll', () => {
@@ -29,6 +31,16 @@ describe('addAll', () => {
2931
expect(output[0].cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu')
3032
expect(output[1].cid.toString()).to.equal('bafkreidmuy2n45xj3cdknzprtzo2uvgm3hak6mzy5sllxty457agsftd34')
3133
})
34+
35+
it('recursively adds a directory', async function () {
36+
if (!isNode) {
37+
return this.skip()
38+
}
39+
40+
const res = await last(fs.addAll(globSource('./test/fixtures', 'files/**/*')))
41+
42+
expect(res?.cid.toString()).to.equal('bafybeievhllpjjjbyg53g74wcl5hckdccjjj7zgtexqcacjegoduegnkyu')
43+
})
3244
})
3345

3446
describe('addBytes', () => {
@@ -82,6 +94,12 @@ describe('addFile', () => {
8294

8395
expect(cid.toString()).to.equal('bafkreiaixnpf23vkyecj5xqispjq5ubcwgsntnnurw2bjby7khe4wnjihu')
8496
})
97+
98+
it('adds a file from a URL', async () => {
99+
const cid = await fs.addFile(urlSource(new URL(`${process.env.ECHO_SERVER}/download?data=hello-world`)))
100+
101+
expect(cid.toString()).to.equal('bafkreifpuj5ujvb3aku75ja5cphnylsac3h47b6f3p4zbzmtm2nkrtrinu')
102+
})
85103
})
86104

87105
describe('addDirectory', () => {

packages/unixfs/test/fixtures/files/another-dir/another-nested-dir/other.txt

Whitespace-only changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
hello

packages/unixfs/test/fixtures/files/dir/.hidden.txt

Whitespace-only changes.

packages/unixfs/test/fixtures/files/dir/file-1.txt

Whitespace-only changes.

packages/unixfs/test/fixtures/files/dir/file-2.js

Whitespace-only changes.

packages/unixfs/test/fixtures/files/dir/file-3.css

Whitespace-only changes.

packages/unixfs/test/fixtures/files/dir/nested-dir/other.txt

Whitespace-only changes.

packages/unixfs/test/fixtures/files/file-0.html

Whitespace-only changes.

0 commit comments

Comments
 (0)