diff --git a/README.md b/README.md index c4cf9e97..395aaaa1 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ gsub.on('fruit', (data) => { }) gsub.subscribe('fruit') -gsub.publish('fruit', new Buffer('banana')) +gsub.publish('fruit', new TextEncoder().encode('banana')) ``` ## API diff --git a/benchmarks/benchmarks.js b/benchmarks/benchmarks.js index a8436835..297ca90f 100644 --- a/benchmarks/benchmarks.js +++ b/benchmarks/benchmarks.js @@ -25,7 +25,7 @@ const suite = new Benchmark.Suite('gossipsub') suite .add('publish and receive', (deferred) => { peers[1].gs.once('Z', (msg) => deferred.resolve(msg)) - peers[0].gs.publish('Z', Buffer.alloc(1024)) + peers[0].gs.publish('Z', new Uint8Array(1024)) }, { defer: true }) diff --git a/package.json b/package.json index 2666ef83..cddd9155 100644 --- a/package.json +++ b/package.json @@ -36,15 +36,14 @@ "lint" ], "dependencies": { - "buffer": "^5.6.0", "debug": "^4.1.1", "err-code": "^2.0.0", "it-length-prefixed": "^3.0.0", "it-pipe": "^1.0.1", - "libp2p-pubsub": "~0.5.2", + "libp2p-pubsub": "^0.6.0", "p-map": "^4.0.0", - "peer-id": "~0.13.12", - "protons": "^1.0.1", + "peer-id": "^0.14.0", + "protons": "^2.0.0", "time-cache": "^0.3.0" }, "devDependencies": { @@ -52,7 +51,7 @@ "@types/mocha": "^7.0.2", "@typescript-eslint/eslint-plugin": "^3.0.2", "@typescript-eslint/parser": "^3.0.2", - "aegir": "^21.10.2", + "aegir": "^25.0.0", "benchmark": "^2.1.4", "chai": "^4.2.0", "chai-spies": "^1.0.0", @@ -72,7 +71,8 @@ "p-wait-for": "^3.1.0", "promisify-es6": "^1.0.3", "sinon": "^9.0.2", - "typescript": "^3.9.3" + "typescript": "^3.9.3", + "uint8arrays": "^1.1.0" }, "contributors": [ "Cayman ", diff --git a/test/2-nodes.spec.js b/test/2-nodes.spec.js index 7c584efc..070cc1c2 100644 --- a/test/2-nodes.spec.js +++ b/test/2-nodes.spec.js @@ -1,11 +1,12 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) chai.use(require('chai-spies')) const expect = chai.expect +const uint8ArrayFromString = require('uint8arrays/from-string') + const { multicodec } = require('../src') @@ -151,7 +152,7 @@ describe('2 nodes', () => { const promise = new Promise((resolve) => nodes[1].once(topic, resolve)) nodes[0].once(topic, (m) => shouldNotHappen) - nodes[0].publish(topic, Buffer.from('hey')) + nodes[0].publish(topic, uint8ArrayFromString('hey')) const msg = await promise @@ -165,7 +166,7 @@ describe('2 nodes', () => { const promise = new Promise((resolve) => nodes[0].once(topic, resolve)) nodes[1].once(topic, shouldNotHappen) - nodes[1].publish(topic, Buffer.from('banana')) + nodes[1].publish(topic, uint8ArrayFromString('banana')) const msg = await promise @@ -185,7 +186,7 @@ describe('2 nodes', () => { function receivedMsg (msg) { expect(msg.data.toString()).to.equal('banana') expect(msg.from).to.be.eql(nodes[1].peerId.toB58String()) - expect(Buffer.isBuffer(msg.seqno)).to.be.true() + expect(msg.seqno).to.be.a('Uint8Array') expect(msg.topicIDs).to.be.eql([topic]) if (++counter === 10) { @@ -196,7 +197,7 @@ describe('2 nodes', () => { } Array.from({ length: 10 }).forEach(() => { - nodes[1].publish(topic, Buffer.from('banana')) + nodes[1].publish(topic, uint8ArrayFromString('banana')) }) }) @@ -210,7 +211,7 @@ describe('2 nodes', () => { function receivedMsg (msg) { expect(msg.data.toString()).to.equal('banana') expect(msg.from).to.be.eql(nodes[1].peerId.toB58String()) - expect(Buffer.isBuffer(msg.seqno)).to.be.true() + expect(msg.seqno).to.be.a('Uint8Array') expect(msg.topicIDs).to.be.eql([topic]) if (++counter === 10) { @@ -222,7 +223,7 @@ describe('2 nodes', () => { const msgs = [] Array.from({ length: 10 }).forEach(() => { - msgs.push(Buffer.from('banana')) + msgs.push(uint8ArrayFromString('banana')) }) nodes[1].publish(topic, msgs) }) @@ -281,8 +282,8 @@ describe('2 nodes', () => { }, 100) }) - nodes[1].publish('Z', Buffer.from('banana')) - nodes[0].publish('Z', Buffer.from('banana')) + nodes[1].publish('Z', uint8ArrayFromString('banana')) + nodes[0].publish('Z', uint8ArrayFromString('banana')) try { await promise diff --git a/test/emit-self.spec.js b/test/emit-self.spec.js index 8c2b15a8..a52ff32e 100644 --- a/test/emit-self.spec.js +++ b/test/emit-self.spec.js @@ -1,10 +1,11 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') + const chai = require('chai') chai.use(require('dirty-chai')) chai.use(require('chai-spies')) const expect = chai.expect +const uint8ArrayFromString = require('uint8arrays/from-string') const { createGossipsub, @@ -29,7 +30,7 @@ describe('emit self', () => { it('should emit to self on publish', async () => { const promise = new Promise((resolve) => gossipsub.once(topic, resolve)) - gossipsub.publish(topic, Buffer.from('hey')) + gossipsub.publish(topic, uint8ArrayFromString('hey')) await promise }) @@ -46,7 +47,7 @@ describe('emit self', () => { it('should emit to self on publish', async () => { gossipsub.once(topic, (m) => shouldNotHappen) - gossipsub.publish(topic, Buffer.from('hey')) + gossipsub.publish(topic, uint8ArrayFromString('hey')) // Wait 1 second to guarantee that self is not noticed await new Promise((resolve) => setTimeout(() => resolve(), 1000)) diff --git a/test/floodsub.spec.js b/test/floodsub.spec.js index 831c1919..ec9e41ce 100644 --- a/test/floodsub.spec.js +++ b/test/floodsub.spec.js @@ -1,9 +1,9 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) +const uint8ArrayFromString = require('uint8arrays/from-string') const expect = chai.expect const times = require('lodash/times') @@ -228,7 +228,7 @@ describe('gossipsub fallbacks to floodsub', () => { const promise = new Promise((resolve) => nodeFs.once(topic, resolve)) nodeGs.once(topic, (m) => shouldNotHappen) - nodeGs.publish(topic, Buffer.from('hey')) + nodeGs.publish(topic, uint8ArrayFromString('hey')) promise.then((msg) => { expect(msg.data.toString()).to.equal('hey') @@ -242,7 +242,7 @@ describe('gossipsub fallbacks to floodsub', () => { it('Publish to a topic - nodeFs', async () => { const promise = new Promise((resolve) => nodeGs.once(topic, resolve)) - nodeFs.publish(topic, Buffer.from('banana')) + nodeFs.publish(topic, uint8ArrayFromString('banana')) const msg = await promise @@ -263,7 +263,7 @@ describe('gossipsub fallbacks to floodsub', () => { function receivedMsg (msg) { expect(msg.data.toString()).to.equal('banana ' + counter) expect(msg.from).to.be.eql(nodeGs.peerId.toB58String()) - expect(Buffer.isBuffer(msg.seqno)).to.be.true() + expect(msg.seqno).to.be.a('Uint8Array') expect(msg.topicIDs).to.be.eql([topic]) if (++counter === 10) { @@ -273,7 +273,7 @@ describe('gossipsub fallbacks to floodsub', () => { } } - times(10, (index) => nodeGs.publish(topic, Buffer.from('banana ' + index))) + times(10, (index) => nodeGs.publish(topic, uint8ArrayFromString('banana ' + index))) }) it('Publish 10 msg to a topic as array', (done) => { @@ -290,7 +290,7 @@ describe('gossipsub fallbacks to floodsub', () => { function receivedMsg (msg) { expect(msg.data.toString()).to.equal('banana ' + counter) expect(msg.from).to.be.eql(nodeGs.peerId.toB58String()) - expect(Buffer.isBuffer(msg.seqno)).to.be.true() + expect(msg.seqno).to.be.a('Uint8Array') expect(msg.topicIDs).to.be.eql([topic]) if (++counter === 10) { @@ -301,7 +301,7 @@ describe('gossipsub fallbacks to floodsub', () => { } const msgs = [] - times(10, (index) => msgs.push(Buffer.from('banana ' + index))) + times(10, (index) => msgs.push(uint8ArrayFromString('banana ' + index))) nodeGs.publish(topic, msgs) }) }) @@ -384,8 +384,8 @@ describe('gossipsub fallbacks to floodsub', () => { }, 100) }) - nodeFs.publish('Z', Buffer.from('banana')) - nodeGs.publish('Z', Buffer.from('banana')) + nodeFs.publish('Z', uint8ArrayFromString('banana')) + nodeGs.publish('Z', uint8ArrayFromString('banana')) try { await promise diff --git a/test/gossip-incoming.spec.js b/test/gossip-incoming.spec.js index 8d4cf357..96a38212 100644 --- a/test/gossip-incoming.spec.js +++ b/test/gossip-incoming.spec.js @@ -1,11 +1,12 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) chai.use(require('chai-spies')) const expect = chai.expect +const uint8ArrayFromString = require('uint8arrays/from-string') + const { GossipsubID: multicodec } = require('../src/constants') const { createGossipsubConnectedNodes } = require('./utils') @@ -43,7 +44,7 @@ describe('gossip incoming', () => { const promise = new Promise((resolve) => nodes[2].once(topic, resolve)) nodes[0].once(topic, (m) => shouldNotHappen) - nodes[0].publish(topic, Buffer.from('hey')) + nodes[0].publish(topic, uint8ArrayFromString('hey')) const msg = await promise @@ -80,7 +81,7 @@ describe('gossip incoming', () => { it('should not gossip incoming messages', async () => { nodes[2].once(topic, (m) => shouldNotHappen) - nodes[0].publish(topic, Buffer.from('hey')) + nodes[0].publish(topic, uint8ArrayFromString('hey')) await new Promise((resolve) => setTimeout(resolve, 1000)) diff --git a/test/gossip.js b/test/gossip.js index cf001f67..9437ab08 100644 --- a/test/gossip.js +++ b/test/gossip.js @@ -1,9 +1,9 @@ 'use strict' /* eslint-env mocha */ -const { Buffer } = require('buffer') const { expect } = require('chai') const sinon = require('sinon') +const uint8ArrayFromString = require('uint8arrays/from-string') const { GossipsubID: multicodec, GossipsubDhi } = require('../src/constants') const { @@ -42,7 +42,7 @@ describe('gossip', () => { // set spy sinon.spy(nodeA, 'log') - await nodeA.publish(topic, Buffer.from('hey')) + await nodeA.publish(topic, uint8ArrayFromString('hey')) await new Promise((resolve) => nodeA.once('gossipsub:heartbeat', resolve)) @@ -84,7 +84,7 @@ describe('gossip', () => { // manually add control message to be sent to peerB nodeA.control.set(peerB, { graft: [{ topicID: topic }] }) - await nodeA.publish(topic, Buffer.from('hey')) + await nodeA.publish(topic, uint8ArrayFromString('hey')) await new Promise((resolve) => nodeA.once('gossipsub:heartbeat', resolve)) expect(nodeB.log.callCount).to.be.gt(1) diff --git a/test/messageCache.spec.js b/test/messageCache.spec.js index 5404efbb..c2a57f9f 100644 --- a/test/messageCache.spec.js +++ b/test/messageCache.spec.js @@ -1,13 +1,14 @@ /* eslint-env mocha */ /* eslint-disable no-unused-expressions */ 'use strict' -const { Buffer } = require('buffer') + const chai = require('chai') const dirtyChai = require('dirty-chai') chai.use(dirtyChai) const chaiSpies = require('chai-spies') chai.use(chaiSpies) const expect = chai.expect +const uint8ArrayFromString = require('uint8arrays/from-string') const { MessageCache } = require('../src/messageCache') const { utils } = require('libp2p-pubsub') @@ -24,7 +25,7 @@ describe('Testing Message Cache Operations', () => { const makeTestMessage = (n) => { return { from: 'test', - data: Buffer.from(n.toString()), + data: uint8ArrayFromString(n.toString()), seqno: utils.randomSeqno(), topicIDs: ['test'] } diff --git a/test/multiple-nodes.spec.js b/test/multiple-nodes.spec.js index 9b7bc6e1..e04270d7 100644 --- a/test/multiple-nodes.spec.js +++ b/test/multiple-nodes.spec.js @@ -1,11 +1,12 @@ /* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { Buffer } = require('buffer') + const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect const promisify = require('promisify-es6') +const uint8ArrayFromString = require('uint8arrays/from-string') const { GossipsubID: multicodec } = require('../src/constants') const { @@ -192,7 +193,7 @@ describe('multiple nodes (more than 2)', () => { let msgB = new Promise((resolve) => b.once('Z', resolve)) let msgC = new Promise((resolve) => c.once('Z', resolve)) - a.publish('Z', Buffer.from('hey')) + a.publish('Z', uint8ArrayFromString('hey')) msgB = await msgB msgC = await msgC @@ -222,7 +223,7 @@ describe('multiple nodes (more than 2)', () => { }) }) - a.publish('Z', [Buffer.from('hey'), Buffer.from('hey')]) + a.publish('Z', [uint8ArrayFromString('hey'), uint8ArrayFromString('hey')]) msgB = await msgB msgC = await msgC @@ -318,7 +319,7 @@ describe('multiple nodes (more than 2)', () => { let msgA = new Promise((resolve) => a.once('Z', resolve)) let msgC = new Promise((resolve) => c.once('Z', resolve)) - b.publish('Z', Buffer.from('hey')) + b.publish('Z', uint8ArrayFromString('hey')) msgA = await msgA msgC = await msgC @@ -460,7 +461,7 @@ describe('multiple nodes (more than 2)', () => { let msgE = new Promise((resolve) => e.once('Z', resolve)) const msg = 'hey from c' - c.publish('Z', Buffer.from(msg)) + c.publish('Z', uint8ArrayFromString(msg)) msgA = await msgA msgB = await msgB diff --git a/test/pubsub.spec.js b/test/pubsub.spec.js index 4b5bd93b..f89c5220 100644 --- a/test/pubsub.spec.js +++ b/test/pubsub.spec.js @@ -1,11 +1,13 @@ 'use strict' /* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 5] */ -const { Buffer } = require('buffer') + const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect const sinon = require('sinon') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayEquals = require('uint8arrays/equals') const { utils } = require('libp2p-pubsub') const { @@ -30,7 +32,7 @@ describe('Pubsub', () => { it('should sign messages on publish', async () => { sinon.spy(gossipsub, '_publish') - await gossipsub.publish('signing-topic', Buffer.from('hello')) + await gossipsub.publish('signing-topic', uint8ArrayFromString('hello')) // Get the first message sent to _publish, and validate it const signedMessage = gossipsub._publish.getCall(0).lastArg[0] @@ -51,7 +53,7 @@ describe('Pubsub', () => { subscriptions: [], msgs: [{ from: gossipsub.peerId.id, - data: Buffer.from('an unsigned message'), + data: uint8ArrayFromString('an unsigned message'), seqno: utils.randomSeqno(), topicIDs: [topic] }] @@ -74,7 +76,7 @@ describe('Pubsub', () => { const topic = 'my-topic' const signedMessage = await gossipsub._buildMessage({ from: gossipsub.peerId.id, - data: Buffer.from('an unsigned message'), + data: uint8ArrayFromString('an unsigned message'), seqno: utils.randomSeqno(), topicIDs: [topic] }) @@ -105,7 +107,7 @@ describe('Pubsub', () => { subscriptions: [], msgs: [{ from: gossipsub.peerId.id, - data: Buffer.from('an unsigned message'), + data: uint8ArrayFromString('an unsigned message'), seqno: utils.randomSeqno(), topicIDs: [topic] }] @@ -134,7 +136,7 @@ describe('Pubsub', () => { // Set a trivial topic validator gossipsub.topicValidators.set(filteredTopic, (topic, peer, message) => { - return message.data.equals(Buffer.from('a message')) + return uint8ArrayEquals(message.data, uint8ArrayFromString('a message')) }) // valid case @@ -142,7 +144,7 @@ describe('Pubsub', () => { subscriptions: [], msgs: [{ from: gossipsub.peerId.id, - data: Buffer.from('a message'), + data: uint8ArrayFromString('a message'), seqno: utils.randomSeqno(), topicIDs: [filteredTopic] }] @@ -158,7 +160,7 @@ describe('Pubsub', () => { subscriptions: [], msgs: [{ from: gossipsub.peerId.id, - data: Buffer.from('a different message'), + data: uint8ArrayFromString('a different message'), seqno: utils.randomSeqno(), topicIDs: [filteredTopic] }] @@ -177,7 +179,7 @@ describe('Pubsub', () => { subscriptions: [], msgs: [{ from: gossipsub.peerId.id, - data: Buffer.from('a different message'), + data: uint8ArrayFromString('a different message'), seqno: utils.randomSeqno(), topicIDs: [filteredTopic] }] diff --git a/ts/message/index.ts b/ts/message/index.ts index fb984c12..c7c8c095 100644 --- a/ts/message/index.ts +++ b/ts/message/index.ts @@ -31,17 +31,17 @@ export interface Message { * * Note: This is not necessarily the peer who sent the RPC this message is contained in */ - from?: Buffer + from?: Uint8Array /** * Opaque blob of data */ - data?: Buffer + data?: Uint8Array /** * 64-bit big-endian uint * * No two messages on a topic from the same peer should have the same seqno value */ - seqno?: Buffer + seqno?: Uint8Array /** * Set of topics being published to */ @@ -52,11 +52,11 @@ export interface Message { * The signature is computed over the marshalled message protobuf excluding the key field * The protobuf bloc is prefixed by the string `libp2p-pubsub:` before signing */ - signature?: Buffer + signature?: Uint8Array /** * Signing key */ - key?: Buffer + key?: Uint8Array } type Overwrite = { @@ -126,8 +126,8 @@ export interface RPC { } interface ProtoCodec { - encode(obj: T): Buffer - decode(buf: Buffer): T + encode(obj: T): Uint8Array + decode(buf: Uint8Array): T } export const RPCCodec = rpcProto.RPC as ProtoCodec diff --git a/ts/peer.ts b/ts/peer.ts index 85791aef..c891eb2b 100644 --- a/ts/peer.ts +++ b/ts/peer.ts @@ -10,10 +10,10 @@ export interface Peer { protocols: string[] conn: Connection topics: Set - stream: Pushable + stream: Pushable readonly isConnected: boolean readonly isWritable: boolean - write (buf: Buffer): void + write (buf: Uint8Array): void attachConnection (conn: Connection): void sendSubscriptions (topics: string[]): void sendUnsubscriptions (topics: string[]): void diff --git a/ts/pubsub.js b/ts/pubsub.js index 6fb0d518..48e36681 100644 --- a/ts/pubsub.js +++ b/ts/pubsub.js @@ -1,7 +1,6 @@ 'use strict' const errcode = require('err-code') -const { Buffer } = require('buffer') const PeerId = require('peer-id') const pipe = require('it-pipe') @@ -112,7 +111,7 @@ class BasicPubSub extends Pubsub { lp.decode(), async (source) => { for await (const data of source) { - const rpcMsgBuf = Buffer.isBuffer(data) ? data : data.slice() + const rpcMsgBuf = data instanceof Uint8Array ? data : data.slice() const rpcMsg = this._decodeRpc(rpcMsgBuf) this._processRpc(idB58Str, peer, rpcMsg) @@ -125,11 +124,11 @@ class BasicPubSub extends Pubsub { } /** - * Decode a buffer into an RPC object + * Decode a Uint8Array into an RPC object * * Override to use an extended protocol-specific protobuf decoder * - * @param {Buffer} buf + * @param {Uint8Array} buf * @returns {RPC} */ _decodeRpc (buf) {