fix: replace node buffers with uint8arrays (#730)

* fix: replace node buffers with uint8arrays

Upgrades all deps and replaces all use of node Buffers with Uint8Arrays

BREAKING CHANGES:

- All deps used by this module now use Uint8Arrays in place of node Buffers

* chore: browser fixes

* chore: remove .only

* chore: stringify uint8array before parsing

* chore: update interop suite

* chore: remove ts from build command

* chore: update deps

* fix: update records to use uint8array

* chore: fix lint

* chore: update deps

Co-authored-by: Jacob Heun <jacobheun@gmail.com>

Author: achingbrain

.aegir.js

@@ -49,5 +49,11 @@ module.exports = {
   hooks: {
     pre: before,
     post: after
+  },
+  webpack: {
+    node: {
+      // needed by bcrypto
+      Buffer: true
+    }
   }
 }

doc/API.md

@@ -585,7 +585,7 @@ Writes a value to a key in the DHT.
 | Name | Type | Description |
 |------|------|-------------|
 | key | `string` | key to add to the dht |
-| value | `Buffer` | value to add to the dht |
+| value | `Uint8Array` | value to add to the dht |
 | [options] | `object` | put options |
 | [options.minPeers] | `number` | minimum number of peers required to successfully put (default: closestPeers.length) |
 
@@ -600,7 +600,7 @@ Writes a value to a key in the DHT.
 ```js
 // ...
 const key = '/key'
-const value = Buffer.from('oh hello there')
+const value = uint8ArrayFromString('oh hello there')
 
 await libp2p.contentRouting.put(key, value)
 ```
@@ -623,7 +623,7 @@ Queries the DHT for a value stored for a given key.
 
 | Type | Description |
 |------|-------------|
-| `Promise<Buffer>` | Value obtained from the DHT |
+| `Promise<Uint8Array>` | Value obtained from the DHT |
 
 #### Example
 
@@ -653,7 +653,7 @@ Queries the DHT for the n values stored for the given key (without sorting).
 
 | Type | Description |
 |------|-------------|
-| `Promise<Array<{from: PeerId, val: Buffer}>>` | Array of records obtained from the DHT |
+| `Promise<Array<{from: PeerId, val: Uint8Array}>>` | Array of records obtained from the DHT |
 
 #### Example
 
@@ -970,7 +970,7 @@ Delete the provided peer from the book.
 ```js
 peerStore.metadataBook.delete(peerId)
 // false
-peerStore.metadataBook.set(peerId, 'nickname', Buffer.from('homePeer'))
+peerStore.metadataBook.set(peerId, 'nickname', uint8ArrayFromString('homePeer'))
 peerStore.metadataBook.delete(peerId)
 // true
 ```
@@ -999,7 +999,7 @@ Deletes the provided peer metadata key-value pair from the book.
 ```js
 peerStore.metadataBook.deleteValue(peerId, 'location')
 // false
-peerStore.metadataBook.set(peerId, 'location', Buffer.from('Berlin'))
+peerStore.metadataBook.set(peerId, 'location', uint8ArrayFromString('Berlin'))
 peerStore.metadataBook.deleteValue(peerId, 'location')
 // true
 ```
@@ -1020,14 +1020,14 @@ Get the known metadata of a provided peer.
 
 | Type | Description |
 |------|-------------|
-| `Map<string, Buffer>` | Peer Metadata |
+| `Map<string, Uint8Array>` | Peer Metadata |
 
 #### Example
 
 ```js
 peerStore.metadataBook.get(peerId)
 // undefined
-peerStore.metadataBook.set(peerId, 'location', Buffer.from('Berlin'))
+peerStore.metadataBook.set(peerId, 'location', uint8ArrayFromString('Berlin'))
 peerStore.metadataBook.get(peerId)
 // Metadata Map
 ```
@@ -1049,14 +1049,14 @@ Get specific metadata of a provided peer.
 
 | Type | Description |
 |------|-------------|
-| `Map<string, Buffer>` | Peer Metadata |
+| `Map<string, Uint8Array>` | Peer Metadata |
 
 #### Example
 
 ```js
 peerStore.metadataBook.getValue(peerId, 'location')
 // undefined
-peerStore.metadataBook.set(peerId, 'location', Buffer.from('Berlin'))
+peerStore.metadataBook.set(peerId, 'location', uint8ArrayFromString('Berlin'))
 peerStore.metadataBook.getValue(peerId, 'location')
 // Metadata Map
 ```
@@ -1073,7 +1073,7 @@ Set known metadata of a given `peerId`.
 |------|------|-------------|
 | peerId | [`PeerId`][peer-id] | peerId to set |
 | key | `string` | key of the metadata value to store |
-| value | `Buffer` | metadata value to store |
+| value | `Uint8Array` | metadata value to store |
 
 #### Returns
 
@@ -1084,7 +1084,7 @@ Set known metadata of a given `peerId`.
 #### Example
 
 ```js
-peerStore.metadataBook.set(peerId, 'location', Buffer.from('Berlin'))
+peerStore.metadataBook.set(peerId, 'location', uint8ArrayFromString('Berlin'))
 ```
 
 ### peerStore.protoBook.delete
@@ -1308,7 +1308,7 @@ Publishes messages to the given topics.
 | Name | Type | Description |
 |------|------|-------------|
 | topic | `string` | topic to publish |
-| data | `Buffer` | data to publish  |
+| data | `Uint8Array` | data to publish  |
 
 #### Returns
 
@@ -1320,7 +1320,7 @@ Publishes messages to the given topics.
 
 ```js
 const topic = 'topic'
-const data = Buffer.from('data')
+const data = uint8ArrayFromString('data')
 
 await libp2p.pubsub.publish(topic, data)
 ```
@@ -1336,7 +1336,7 @@ Subscribes the given handler to a pubsub topic.
 | Name | Type | Description |
 |------|------|-------------|
 | topic | `string` | topic to subscribe |
-| handler | `function({ from: string, data: Buffer, seqno: Buffer, topicIDs: Array<string>, signature: Buffer, key: Buffer })` | handler for new data on topic |
+| handler | `function({ from: string, data: Uint8Array, seqno: Uint8Array, topicIDs: Array<string>, signature: Uint8Array, key: Uint8Array })` | handler for new data on topic |
 
 #### Returns
 
@@ -1679,19 +1679,19 @@ Encrypt protected data using the Cryptographic Message Syntax (CMS).
 | Name | Type | Description |
 |------|------|-------------|
 | name | `string` | The local key name. |
-| data | `Buffer` | The data to encrypt. |
+| data | `Uint8Array` | The data to encrypt. |
 
 #### Returns
 
 | Type | Description |
 |------|-------------|
-| `Promise<Buffer>` | Encrypted data as a PKCS #7 message in DER. |
+| `Promise<Uint8Array>` | Encrypted data as a PKCS #7 message in DER. |
 
 #### Example
 
 ```js
 const keyInfo = await libp2p.keychain.createKey('keyTest', 'rsa', 4096)
-const enc = await libp2p.keychain.cms.encrypt('keyTest', Buffer.from('data'))
+const enc = await libp2p.keychain.cms.encrypt('keyTest', uint8ArrayFromString('data'))
 ```
 
 ### keychain.cms.decrypt
@@ -1711,13 +1711,13 @@ The keychain must contain one of the keys used to encrypt the data.  If none of
 
 | Type | Description |
 |------|-------------|
-| `Promise<Buffer>` | Decrypted data. |
+| `Promise<Uint8Array>` | Decrypted data. |
 
 #### Example
 
 ```js
 const keyInfo = await libp2p.keychain.createKey('keyTest', 'rsa', 4096)
-const enc = await libp2p.keychain.cms.encrypt('keyTest', Buffer.from('data'))
+const enc = await libp2p.keychain.cms.encrypt('keyTest', uint8ArrayFromString('data'))
 const decData = await libp2p.keychain.cms.decrypt(enc)
 ```
 

examples/chat/src/stream.js

@@ -3,6 +3,7 @@
 
 const pipe = require('it-pipe')
 const lp = require('it-length-prefixed')
+const uint8ArrayToString = require('uint8arrays/to-string')
 
 function stdinToStream(stream) {
   // Read utf-8 from stdin
@@ -28,7 +29,7 @@ function streamToConsole(stream) {
       // For each chunk of data
       for await (const msg of source) {
         // Output the data as a utf8 string
-        console.log('> ' + msg.toString('utf8').replace('\n', ''))
+        console.log('> ' + uint8ArrayToString(msg).replace('\n', ''))
       }
     }
   )

examples/pnet/index.js

@@ -1,18 +1,17 @@
 /* eslint no-console: ["off"] */
 'use strict'
 
-const { Buffer } = require('buffer')
 const { generate } = require('libp2p/src/pnet')
 const privateLibp2pNode = require('./libp2p-node')
 
 const pipe = require('it-pipe')
 
-// Create a buffer and write the swarm key to it
-const swarmKey = Buffer.alloc(95)
+// Create a Uint8Array and write the swarm key to it
+const swarmKey = new Uint8Array(95)
 generate(swarmKey)
 
 // This key is for testing a different key not working
-const otherSwarmKey = Buffer.alloc(95)
+const otherSwarmKey = new Uint8Array(95)
 generate(otherSwarmKey)
 
 ;(async () => {

examples/pnet/libp2p-node.js

@@ -11,7 +11,7 @@ const Protector = require('libp2p/src/pnet')
  * privateLibp2pNode returns a libp2p node function that will use the swarm
  * key with the given `swarmKey` to create the Protector
  *
- * @param {Buffer} swarmKey
+ * @param {Uint8Array} swarmKey
  * @returns {Promise<libp2p>} Returns a libp2pNode function for use in IPFS creation
  */
 const privateLibp2pNode = async (swarmKey) => {

examples/pubsub/1.js

@@ -1,13 +1,13 @@
 /* eslint-disable no-console */
 'use strict'
 
-const { Buffer } = require('buffer')
 const Libp2p = require('../../')
 const TCP = require('libp2p-tcp')
 const Mplex = require('libp2p-mplex')
 const { NOISE } = require('libp2p-noise')
 const SECIO = require('libp2p-secio')
 const Gossipsub = require('libp2p-gossipsub')
+const uint8ArrayFromString = require('uint8arrays/from-string')
 
 const createNode = async () => {
   const node = await Libp2p.create({
@@ -48,6 +48,6 @@ const createNode = async () => {
 
   // node2 publishes "news" every second
   setInterval(() => {
-    node2.pubsub.publish(topic, Buffer.from('Bird bird bird, bird is the word!'))
+    node2.pubsub.publish(topic, uint8ArrayFromString('Bird bird bird, bird is the word!'))
   }, 1000)
 })()

examples/pubsub/README.md

@@ -57,7 +57,7 @@ await node2.pubsub.subscribe(topic, (msg) => {
 
 // node2 publishes "news" every second
 setInterval(() => {
-  node2.pubsub.publish(topic, Buffer.from('Bird bird bird, bird is the word!'))
+  node2.pubsub.publish(topic, uint8ArrayFromString('Bird bird bird, bird is the word!'))
 }, 1000)
 ```
 

examples/pubsub/message-filtering/1.js

@@ -1,13 +1,13 @@
 /* eslint-disable no-console */
 'use strict'
 
-const { Buffer } = require('buffer')
 const Libp2p = require('../../../')
 const TCP = require('libp2p-tcp')
 const Mplex = require('libp2p-mplex')
 const { NOISE } = require('libp2p-noise')
 const SECIO = require('libp2p-secio')
 const Gossipsub = require('libp2p-gossipsub')
+const uint8ArrayFromString = require('uint8arrays/from-string')
 
 const createNode = async () => {
   const node = await Libp2p.create({
@@ -73,7 +73,7 @@ const createNode = async () => {
   // car is not a fruit !
   setInterval(() => {
     console.log('############## fruit ' + myFruits[count] + ' ##############')
-    node1.pubsub.publish(topic, Buffer.from(myFruits[count]))
+    node1.pubsub.publish(topic, uint8ArrayFromString(myFruits[count]))
     count++
     if (count == myFruits.length) {
       count = 0

examples/pubsub/message-filtering/README.md

@@ -79,7 +79,7 @@ const myFruits = ['banana', 'apple', 'car', 'orange'];
 
 setInterval(() => {
   console.log('############## fruit ' + myFruits[count] + ' ##############')
-  node1.pubsub.publish(topic, Buffer.from(myFruits[count]))
+  node1.pubsub.publish(topic, new TextEncoder().encode(myFruits[count]))
   count++
   if (count == myFruits.length) {
     count = 0