From 25488853ef6a7983b803a32bc17d2c65f86e94e4 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 16 Sep 2020 16:43:09 +0200 Subject: [PATCH 01/17] feat: auto relay (#723) * feat: auto relay * fix: leverage protoBook events to ask relay peers if they support hop * chore: refactor disconnect * chore: do not listen on a relayed conn * chore: tweaks * chore: improve _listenOnAvailableHopRelays logic * chore: default value of 1 to maxListeners on auto-relay --- src/circuit/auto-relay.js | 231 +++++++++++ src/circuit/circuit/hop.js | 27 ++ src/circuit/index.js | 20 +- src/circuit/listener.js | 16 +- src/config.js | 4 + src/identify/index.js | 7 +- src/index.js | 9 +- src/peer-store/address-book.js | 2 +- src/transport-manager.js | 7 +- test/relay/auto-relay.node.js | 455 ++++++++++++++++++++++ test/{dialing => relay}/relay.node.js | 4 +- test/transports/transport-manager.node.js | 4 +- test/transports/transport-manager.spec.js | 2 +- 13 files changed, 762 insertions(+), 26 deletions(-) create mode 100644 src/circuit/auto-relay.js create mode 100644 test/relay/auto-relay.node.js rename test/{dialing => relay}/relay.node.js (97%) diff --git a/src/circuit/auto-relay.js b/src/circuit/auto-relay.js new file mode 100644 index 0000000000..5617e94eff --- /dev/null +++ b/src/circuit/auto-relay.js @@ -0,0 +1,231 @@ +'use strict' + +const debug = require('debug') +const log = debug('libp2p:auto-relay') +log.error = debug('libp2p:auto-relay:error') + +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require('uint8arrays/to-string') +const multiaddr = require('multiaddr') +const PeerId = require('peer-id') + +const { relay: multicodec } = require('./multicodec') +const { canHop } = require('./circuit/hop') + +const circuitProtoCode = 290 +const hopMetadataKey = 'hop_relay' +const hopMetadataValue = 'true' + +class AutoRelay { + /** + * Creates an instance of AutoRelay. + * @constructor + * @param {object} props + * @param {Libp2p} props.libp2p + * @param {number} [props.maxListeners = 1] maximum number of relays to listen. + */ + constructor ({ libp2p, maxListeners = 1 }) { + this._libp2p = libp2p + this._peerId = libp2p.peerId + this._peerStore = libp2p.peerStore + this._connectionManager = libp2p.connectionManager + this._transportManager = libp2p.transportManager + + this.maxListeners = maxListeners + + /** + * @type {Set} + */ + this._listenRelays = new Set() + + this._onProtocolChange = this._onProtocolChange.bind(this) + this._onPeerDisconnected = this._onPeerDisconnected.bind(this) + + this._peerStore.on('change:protocols', this._onProtocolChange) + this._connectionManager.on('peer:disconnect', this._onPeerDisconnected) + } + + /** + * Check if a peer supports the relay protocol. + * If the protocol is not supported, check if it was supported before and remove it as a listen relay. + * If the protocol is supported, check if the peer supports **HOP** and add it as a listener if + * inside the threshold. + * @param {Object} props + * @param {PeerId} props.peerId + * @param {Array} props.protocols + * @return {Promise} + */ + async _onProtocolChange ({ peerId, protocols }) { + const id = peerId.toB58String() + + // Check if it has the protocol + const hasProtocol = protocols.find(protocol => protocol === multicodec) + + // If no protocol, check if we were keeping the peer before as a listenRelay + if (!hasProtocol && this._listenRelays.has(id)) { + this._removeListenRelay(id) + return + } else if (!hasProtocol || this._listenRelays.has(id)) { + return + } + + // If protocol, check if can hop, store info in the metadataBook and listen on it + try { + const connection = this._connectionManager.get(peerId) + + // Do not hop on a relayed connection + if (connection.remoteAddr.protoCodes().includes(circuitProtoCode)) { + log(`relayed connection to ${id} will not be used to hop on`) + return + } + + const supportsHop = await canHop({ connection }) + + if (supportsHop) { + this._peerStore.metadataBook.set(peerId, hopMetadataKey, uint8ArrayFromString(hopMetadataValue)) + await this._addListenRelay(connection, id) + } + } catch (err) { + log.error(err) + } + } + + /** + * Peer disconnects. + * @param {Connection} connection connection to the peer + * @return {void} + */ + _onPeerDisconnected (connection) { + const peerId = connection.remotePeer + const id = peerId.toB58String() + + // Not listening on this relay + if (!this._listenRelays.has(id)) { + return + } + + this._removeListenRelay(id) + } + + /** + * Attempt to listen on the given relay connection. + * @private + * @param {Connection} connection connection to the peer + * @param {string} id peer identifier string + * @return {Promise} + */ + async _addListenRelay (connection, id) { + // Check if already listening on enough relays + if (this._listenRelays.size >= this.maxListeners) { + return + } + + // Create relay listen addr + let listenAddr, remoteMultiaddr + + try { + const remoteAddrs = this._peerStore.addressBook.get(connection.remotePeer) + // TODO: HOP Relays should avoid advertising private addresses! + remoteMultiaddr = remoteAddrs.find(a => a.isCertified).multiaddr // Get first announced address certified + } catch (_) { + log.error(`${id} does not have announced certified multiaddrs`) + return + } + + if (!remoteMultiaddr.protoNames().includes('p2p')) { + listenAddr = `${remoteMultiaddr.toString()}/p2p/${connection.remotePeer.toB58String()}/p2p-circuit` + } else { + listenAddr = `${remoteMultiaddr.toString()}/p2p-circuit` + } + + // Attempt to listen on relay + this._listenRelays.add(id) + + try { + await this._transportManager.listen([multiaddr(listenAddr)]) + // TODO: push announce multiaddrs update + // await this._libp2p.identifyService.pushToPeerStore() + } catch (err) { + log.error(err) + this._listenRelays.delete(id) + } + } + + /** + * Remove listen relay. + * @private + * @param {string} id peer identifier string. + * @return {void} + */ + _removeListenRelay (id) { + if (this._listenRelays.delete(id)) { + // TODO: this should be responsibility of the connMgr + this._listenOnAvailableHopRelays([id]) + } + } + + /** + * Try to listen on available hop relay connections. + * The following order will happen while we do not have enough relays. + * 1. Check the metadata store for known relays, try to listen on the ones we are already connected. + * 2. Dial and try to listen on the peers we know that support hop but are not connected. + * 3. Search the network. + * @param {Array} [peersToIgnore] + * @return {Promise} + */ + async _listenOnAvailableHopRelays (peersToIgnore = []) { + // TODO: The peer redial issue on disconnect should be handled by connection gating + // Check if already listening on enough relays + if (this._listenRelays.size >= this.maxListeners) { + return + } + + const knownHopsToDial = [] + + // Check if we have known hop peers to use and attempt to listen on the already connected + for (const [id, metadataMap] of this._peerStore.metadataBook.data.entries()) { + // Continue to next if listening on this or peer to ignore + if (this._listenRelays.has(id) || peersToIgnore.includes(id)) { + continue + } + + const supportsHop = metadataMap.get(hopMetadataKey) + + // Continue to next if it does not support Hop + if (!supportsHop || uint8ArrayToString(supportsHop) !== hopMetadataValue) { + continue + } + + const peerId = PeerId.createFromCID(id) + const connection = this._connectionManager.get(peerId) + + // If not connected, store for possible later use. + if (!connection) { + knownHopsToDial.push(peerId) + continue + } + + await this._addListenRelay(connection, id) + + // Check if already listening on enough relays + if (this._listenRelays.size >= this.maxListeners) { + return + } + } + + // Try to listen on known peers that are not connected + for (const peerId of knownHopsToDial) { + const connection = await this._libp2p.dial(peerId) + await this._addListenRelay(connection, peerId.toB58String()) + + // Check if already listening on enough relays + if (this._listenRelays.size >= this.maxListeners) { + return + } + } + + // TODO: Try to find relays to hop on the network + } +} + +module.exports = AutoRelay diff --git a/src/circuit/circuit/hop.js b/src/circuit/circuit/hop.js index f497f33a32..114e2768fe 100644 --- a/src/circuit/circuit/hop.js +++ b/src/circuit/circuit/hop.js @@ -116,6 +116,33 @@ module.exports.hop = async function hop ({ throw errCode(new Error(`HOP request failed with code ${response.code}`), Errors.ERR_HOP_REQUEST_FAILED) } +/** + * Performs a CAN_HOP request to a relay peer, in order to understand its capabilities. + * @param {object} options + * @param {Connection} options.connection Connection to the relay + * @returns {Promise} + */ +module.exports.canHop = async function canHop ({ + connection +}) { + // Create a new stream to the relay + const { stream } = await connection.newStream([multicodec.relay]) + // Send the HOP request + const streamHandler = new StreamHandler({ stream }) + streamHandler.write({ + type: CircuitPB.Type.CAN_HOP + }) + + const response = await streamHandler.read() + await streamHandler.close() + + if (response.code !== CircuitPB.Status.SUCCESS) { + return false + } + + return true +} + /** * Creates an unencoded CAN_HOP response based on the Circuits configuration * diff --git a/src/circuit/index.js b/src/circuit/index.js index 15746c907c..705dcdad82 100644 --- a/src/circuit/index.js +++ b/src/circuit/index.js @@ -1,16 +1,18 @@ 'use strict' +const debug = require('debug') +const log = debug('libp2p:circuit') +log.error = debug('libp2p:circuit:error') + const mafmt = require('mafmt') const multiaddr = require('multiaddr') const PeerId = require('peer-id') const withIs = require('class-is') const { CircuitRelay: CircuitPB } = require('./protocol') -const debug = require('debug') -const log = debug('libp2p:circuit') -log.error = debug('libp2p:circuit:error') const toConnection = require('libp2p-utils/src/stream-to-ma-conn') +const AutoRelay = require('./auto-relay') const { relay: multicodec } = require('./multicodec') const createListener = require('./listener') const { handleCanHop, handleHop, hop } = require('./circuit/hop') @@ -35,11 +37,19 @@ class Circuit { this._libp2p = libp2p this.peerId = libp2p.peerId this._registrar.handle(multicodec, this._onProtocol.bind(this)) + + // Create autoRelay if enabled + this._autoRelay = this._options.autoRelay.enabled && new AutoRelay({ libp2p, ...this._options.autoRelay }) } - async _onProtocol ({ connection, stream, protocol }) { + async _onProtocol ({ connection, stream }) { const streamHandler = new StreamHandler({ stream }) const request = await streamHandler.read() + + if (!request) { + return + } + const circuit = this let virtualConnection @@ -163,7 +173,7 @@ class Circuit { // Called on successful HOP and STOP requests this.handler = handler - return createListener(this, options) + return createListener(this._libp2p, options) } /** diff --git a/src/circuit/listener.js b/src/circuit/listener.js index 76870501dc..f8caff0b41 100644 --- a/src/circuit/listener.js +++ b/src/circuit/listener.js @@ -8,13 +8,23 @@ const log = debug('libp2p:circuit:listener') log.err = debug('libp2p:circuit:error:listener') /** - * @param {*} circuit + * @param {Libp2p} libp2p * @returns {Listener} a transport listener */ -module.exports = (circuit) => { +module.exports = (libp2p) => { const listener = new EventEmitter() const listeningAddrs = new Map() + // Remove listeningAddrs when a peer disconnects + libp2p.connectionManager.on('peer:disconnect', (connection) => { + const deleted = listeningAddrs.delete(connection.remotePeer.toB58String()) + + if (deleted) { + // TODO push announce multiaddrs update + // libp2p.identifyService.pushToPeerStore() + } + }) + /** * Add swarm handler and listen for incoming connections * @@ -24,7 +34,7 @@ module.exports = (circuit) => { listener.listen = async (addr) => { const addrString = String(addr).split('/p2p-circuit').find(a => a !== '') - const relayConn = await circuit._dialer.connectToPeer(multiaddr(addrString)) + const relayConn = await libp2p.dial(multiaddr(addrString)) const relayedAddr = relayConn.remoteAddr.encapsulate('/p2p-circuit') listeningAddrs.set(relayConn.remotePeer.toB58String(), relayedAddr) diff --git a/src/config.js b/src/config.js index 1cc0f097b4..eb28e3c734 100644 --- a/src/config.js +++ b/src/config.js @@ -59,6 +59,10 @@ const DefaultConfig = { hop: { enabled: false, active: false + }, + autoRelay: { + enabled: false, + maxListeners: 2 } }, transport: {} diff --git a/src/identify/index.js b/src/identify/index.js index f42a8b6f94..ba49610ffc 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -121,13 +121,12 @@ class IdentifyService { /** * Calls `push` for all peers in the `peerStore` that are connected - * - * @param {PeerStore} peerStore + * @returns {void} */ - pushToPeerStore (peerStore) { + pushToPeerStore () { const connections = [] let connection - for (const peer of peerStore.peers.values()) { + for (const peer of this.peerStore.peers.values()) { if (peer.protocols.includes(MULTICODEC_IDENTIFY_PUSH) && (connection = this.connectionManager.get(peer.id))) { connections.push(connection) } diff --git a/src/index.js b/src/index.js index 85547f4702..b269122032 100644 --- a/src/index.js +++ b/src/index.js @@ -433,7 +433,7 @@ class Libp2p extends EventEmitter { // Only push if libp2p is running if (this.isStarted() && this.identifyService) { - this.identifyService.pushToPeerStore(this.peerStore) + this.identifyService.pushToPeerStore() } } @@ -451,13 +451,14 @@ class Libp2p extends EventEmitter { // Only push if libp2p is running if (this.isStarted() && this.identifyService) { - this.identifyService.pushToPeerStore(this.peerStore) + this.identifyService.pushToPeerStore() } } async _onStarting () { - // Listen on the provided transports - await this.transportManager.listen() + // Listen on the provided transports for the provided addresses + const addrs = this.addressManager.getListenAddrs() + await this.transportManager.listen(addrs) // Start PeerStore await this.peerStore.start() diff --git a/src/peer-store/address-book.js b/src/peer-store/address-book.js index c8fa2ec6f5..88ed327b3d 100644 --- a/src/peer-store/address-book.js +++ b/src/peer-store/address-book.js @@ -270,7 +270,7 @@ class AddressBook extends Book { * * @override * @param {PeerId} peerId - * @returns {Array} + * @returns {Array
|undefined} */ get (peerId) { if (!PeerId.isPeerId(peerId)) { diff --git a/src/transport-manager.js b/src/transport-manager.js index e18841bf02..7570389073 100644 --- a/src/transport-manager.js +++ b/src/transport-manager.js @@ -137,11 +137,10 @@ class TransportManager { * Starts listeners for each listen Multiaddr. * * @async + * @param {Array} addrs addresses to attempt to listen on */ - async listen () { - const addrs = this.libp2p.addressManager.getListenAddrs() - - if (addrs.length === 0) { + async listen (addrs) { + if (!addrs || addrs.length === 0) { log('no addresses were provided for listening, this node is dial only') return } diff --git a/test/relay/auto-relay.node.js b/test/relay/auto-relay.node.js new file mode 100644 index 0000000000..2a4ba20d57 --- /dev/null +++ b/test/relay/auto-relay.node.js @@ -0,0 +1,455 @@ +'use strict' +/* eslint-env mocha */ + +const chai = require('chai') +chai.use(require('dirty-chai')) +const { expect } = chai + +const delay = require('delay') +const pWaitFor = require('p-wait-for') +const sinon = require('sinon') + +const multiaddr = require('multiaddr') +const Libp2p = require('../../src') +const { relay: relayMulticodec } = require('../../src/circuit/multicodec') + +const { createPeerId } = require('../utils/creators/peer') +const baseOptions = require('../utils/base-options') + +const listenAddr = '/ip4/0.0.0.0/tcp/0' + +describe('auto-relay', () => { + describe('basics', () => { + let libp2p + let relayLibp2p + let autoRelay + + beforeEach(async () => { + const peerIds = await createPeerId({ number: 2 }) + // Create 2 nodes, and turn HOP on for the relay + ;[libp2p, relayLibp2p] = peerIds.map((peerId, index) => { + const opts = { + ...baseOptions, + config: { + ...baseOptions.config, + relay: { + hop: { + enabled: index !== 0 + }, + autoRelay: { + enabled: true, + maxListeners: 1 + } + } + } + } + + return new Libp2p({ + ...opts, + addresses: { + listen: [listenAddr] + }, + connectionManager: { + autoDial: false + }, + peerDiscovery: { + autoDial: false + }, + peerId + }) + }) + + autoRelay = libp2p.transportManager._transports.get('Circuit')._autoRelay + + expect(autoRelay.maxListeners).to.eql(1) + }) + + beforeEach(() => { + // Start each node + return Promise.all([libp2p, relayLibp2p].map(libp2p => libp2p.start())) + }) + + afterEach(() => { + // Stop each node + return Promise.all([libp2p, relayLibp2p].map(libp2p => libp2p.stop())) + }) + + it('should ask if node supports hop on protocol change (relay protocol) and add to listen multiaddrs', async () => { + // Spy if a connected peer is being added as listen relay + sinon.spy(autoRelay, '_addListenRelay') + + const originalMultiaddrsLength = relayLibp2p.multiaddrs.length + + // Discover relay + libp2p.peerStore.addressBook.add(relayLibp2p.peerId, relayLibp2p.multiaddrs) + await libp2p.dial(relayLibp2p.peerId) + + // Wait for peer added as listen relay + await pWaitFor(() => autoRelay._addListenRelay.callCount === 1) + expect(autoRelay._listenRelays.size).to.equal(1) + + // Wait for listen multiaddr update + await pWaitFor(() => libp2p.multiaddrs.length === originalMultiaddrsLength + 1) + expect(libp2p.multiaddrs[originalMultiaddrsLength].getPeerId()).to.eql(relayLibp2p.peerId.toB58String()) + + // Peer has relay multicodec + const knownProtocols = libp2p.peerStore.protoBook.get(relayLibp2p.peerId) + expect(knownProtocols).to.include(relayMulticodec) + }) + }) + + describe('flows with 1 listener max', () => { + let libp2p + let relayLibp2p1 + let relayLibp2p2 + let relayLibp2p3 + let autoRelay1 + + beforeEach(async () => { + const peerIds = await createPeerId({ number: 4 }) + // Create 4 nodes, and turn HOP on for the relay + ;[libp2p, relayLibp2p1, relayLibp2p2, relayLibp2p3] = peerIds.map((peerId, index) => { + let opts = baseOptions + + if (index !== 0) { + opts = { + ...baseOptions, + config: { + ...baseOptions.config, + relay: { + hop: { + enabled: true + }, + autoRelay: { + enabled: true, + maxListeners: 1 + } + } + } + } + } + + return new Libp2p({ + ...opts, + addresses: { + listen: [listenAddr] + }, + connectionManager: { + autoDial: false + }, + peerDiscovery: { + autoDial: false + }, + peerId + }) + }) + + autoRelay1 = relayLibp2p1.transportManager._transports.get('Circuit')._autoRelay + + expect(autoRelay1.maxListeners).to.eql(1) + }) + + beforeEach(() => { + // Start each node + return Promise.all([libp2p, relayLibp2p1, relayLibp2p2, relayLibp2p3].map(libp2p => libp2p.start())) + }) + + afterEach(() => { + // Stop each node + return Promise.all([libp2p, relayLibp2p1, relayLibp2p2, relayLibp2p3].map(libp2p => libp2p.stop())) + }) + + it('should ask if node supports hop on protocol change (relay protocol) and add to listen multiaddrs', async () => { + // Spy if a connected peer is being added as listen relay + sinon.spy(autoRelay1, '_addListenRelay') + + // Discover relay + relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs) + + const originalMultiaddrs1Length = relayLibp2p1.multiaddrs.length + const originalMultiaddrs2Length = relayLibp2p2.multiaddrs.length + + await relayLibp2p1.dial(relayLibp2p2.peerId) + + // Wait for peer added as listen relay + await pWaitFor(() => autoRelay1._addListenRelay.callCount === 1) + expect(autoRelay1._listenRelays.size).to.equal(1) + + // Wait for listen multiaddr update + await Promise.all([ + pWaitFor(() => relayLibp2p1.multiaddrs.length === originalMultiaddrs1Length + 1), + pWaitFor(() => relayLibp2p2.multiaddrs.length === originalMultiaddrs2Length + 1) + ]) + expect(relayLibp2p1.multiaddrs[originalMultiaddrs1Length].getPeerId()).to.eql(relayLibp2p2.peerId.toB58String()) + + // Peer has relay multicodec + const knownProtocols = relayLibp2p1.peerStore.protoBook.get(relayLibp2p2.peerId) + expect(knownProtocols).to.include(relayMulticodec) + }) + + it('should be able to dial a peer from its relayed address previously added', async () => { + const originalMultiaddrs1Length = relayLibp2p1.multiaddrs.length + const originalMultiaddrs2Length = relayLibp2p2.multiaddrs.length + + // Discover relay + relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs) + + await relayLibp2p1.dial(relayLibp2p2.peerId) + + // Wait for listen multiaddr update + await Promise.all([ + pWaitFor(() => relayLibp2p1.multiaddrs.length === originalMultiaddrs1Length + 1), + pWaitFor(() => relayLibp2p2.multiaddrs.length === originalMultiaddrs2Length + 1) + ]) + expect(relayLibp2p1.multiaddrs[originalMultiaddrs1Length].getPeerId()).to.eql(relayLibp2p2.peerId.toB58String()) + + // Dial from the other through a relay + const relayedMultiaddr2 = multiaddr(`${relayLibp2p1.multiaddrs[0]}/p2p/${relayLibp2p1.peerId.toB58String()}/p2p-circuit`) + libp2p.peerStore.addressBook.add(relayLibp2p2.peerId, [relayedMultiaddr2]) + + await libp2p.dial(relayLibp2p2.peerId) + }) + + it('should only add maxListeners relayed addresses', async () => { + const originalMultiaddrs1Length = relayLibp2p1.multiaddrs.length + const originalMultiaddrs2Length = relayLibp2p2.multiaddrs.length + + // Spy if a connected peer is being added as listen relay + sinon.spy(autoRelay1, '_addListenRelay') + sinon.spy(autoRelay1._listenRelays, 'add') + + // Discover one relay and connect + relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs) + await relayLibp2p1.dial(relayLibp2p2.peerId) + + expect(relayLibp2p1.connectionManager.size).to.eql(1) + + // Wait for peer added as listen relay + await pWaitFor(() => autoRelay1._addListenRelay.callCount === 1 && autoRelay1._listenRelays.add.callCount === 1) + expect(autoRelay1._listenRelays.size).to.equal(1) + + // Wait for listen multiaddr update + await Promise.all([ + pWaitFor(() => relayLibp2p1.multiaddrs.length === originalMultiaddrs1Length + 1), + pWaitFor(() => relayLibp2p2.multiaddrs.length === originalMultiaddrs2Length + 1) + ]) + expect(relayLibp2p1.multiaddrs[originalMultiaddrs1Length].getPeerId()).to.eql(relayLibp2p2.peerId.toB58String()) + + // Relay2 has relay multicodec + const knownProtocols2 = relayLibp2p1.peerStore.protoBook.get(relayLibp2p2.peerId) + expect(knownProtocols2).to.include(relayMulticodec) + + // Discover an extra relay and connect + relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs) + await relayLibp2p1.dial(relayLibp2p3.peerId) + + // Wait to guarantee the dialed peer is not added as a listen relay + await delay(300) + + expect(autoRelay1._addListenRelay.callCount).to.equal(2) + expect(autoRelay1._listenRelays.add.callCount).to.equal(1) + expect(autoRelay1._listenRelays.size).to.equal(1) + expect(relayLibp2p1.connectionManager.size).to.eql(2) + + // Relay2 has relay multicodec + const knownProtocols3 = relayLibp2p1.peerStore.protoBook.get(relayLibp2p3.peerId) + expect(knownProtocols3).to.include(relayMulticodec) + }) + + it('should not listen on a relayed address if peer disconnects', async () => { + const originalMultiaddrs1Length = relayLibp2p1.multiaddrs.length + + // Discover one relay and connect + relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs) + await relayLibp2p1.dial(relayLibp2p2.peerId) + + // Wait for listenning on the relay + await pWaitFor(() => relayLibp2p1.multiaddrs.length === originalMultiaddrs1Length + 1) + expect(autoRelay1._listenRelays.size).to.equal(1) + expect(relayLibp2p1.multiaddrs[originalMultiaddrs1Length].getPeerId()).to.eql(relayLibp2p2.peerId.toB58String()) + + // Spy if identify push is fired + sinon.spy(relayLibp2p1.identifyService, 'pushToPeerStore') + + // Disconnect from peer used for relay + await relayLibp2p1.hangUp(relayLibp2p2.peerId) + + // Wait for removed listening on the relay + await pWaitFor(() => relayLibp2p1.multiaddrs.length === originalMultiaddrs1Length) + expect(autoRelay1._listenRelays.size).to.equal(0) + // TODO: identify-push expect(relayLibp2p1.identifyService.pushToPeerStore.callCount).to.equal(1) + }) + + it('should try to listen on other connected peers relayed address if one used relay disconnects', async () => { + const originalMultiaddrs1Length = relayLibp2p1.multiaddrs.length + + // Spy if a connected peer is being added as listen relay + sinon.spy(autoRelay1, '_addListenRelay') + sinon.spy(relayLibp2p1.transportManager, 'listen') + + // Discover one relay and connect + relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs) + await relayLibp2p1.dial(relayLibp2p2.peerId) + + // Discover an extra relay and connect + relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs) + await relayLibp2p1.dial(relayLibp2p3.peerId) + + // Wait for both peer to be attempted to added as listen relay + await pWaitFor(() => autoRelay1._addListenRelay.callCount === 1) + expect(autoRelay1._listenRelays.size).to.equal(1) + expect(relayLibp2p1.connectionManager.size).to.equal(2) + + // Wait for listen multiaddr update + await pWaitFor(() => relayLibp2p1.multiaddrs.length === originalMultiaddrs1Length + 1) + expect(relayLibp2p1.multiaddrs[originalMultiaddrs1Length].getPeerId()).to.eql(relayLibp2p2.peerId.toB58String()) + + // Only one will be used for listeninng + expect(relayLibp2p1.transportManager.listen.callCount).to.equal(1) + + // Spy if relay from listen map was removed + sinon.spy(autoRelay1._listenRelays, 'delete') + + // Disconnect from peer used for relay + await relayLibp2p1.hangUp(relayLibp2p2.peerId) + expect(autoRelay1._listenRelays.delete.callCount).to.equal(1) + expect(autoRelay1._addListenRelay.callCount).to.equal(1) + + // Wait for other peer connected to be added as listen addr + await pWaitFor(() => relayLibp2p1.transportManager.listen.callCount === 2) + expect(autoRelay1._listenRelays.size).to.equal(1) + expect(relayLibp2p1.connectionManager.size).to.eql(1) + + // Wait for listen multiaddr update + await pWaitFor(() => relayLibp2p1.multiaddrs.length === originalMultiaddrs1Length + 1) + expect(relayLibp2p1.multiaddrs[originalMultiaddrs1Length].getPeerId()).to.eql(relayLibp2p3.peerId.toB58String()) + }) + + it('should try to listen on stored peers relayed address if one used relay disconnects and there are not enough connected', async () => { + // Spy if a connected peer is being added as listen relay + sinon.spy(autoRelay1, '_addListenRelay') + sinon.spy(relayLibp2p1.transportManager, 'listen') + + // Discover one relay and connect + relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs) + await relayLibp2p1.dial(relayLibp2p2.peerId) + + // Discover an extra relay and connect to gather its Hop support + relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs) + await relayLibp2p1.dial(relayLibp2p3.peerId) + + // Wait for both peer to be attempted to added as listen relay + await pWaitFor(() => autoRelay1._addListenRelay.callCount === 2) + expect(autoRelay1._listenRelays.size).to.equal(1) + expect(relayLibp2p1.connectionManager.size).to.equal(2) + + // Only one will be used for listeninng + expect(relayLibp2p1.transportManager.listen.callCount).to.equal(1) + + // Disconnect not used listen relay + await relayLibp2p1.hangUp(relayLibp2p3.peerId) + + expect(autoRelay1._listenRelays.size).to.equal(1) + expect(relayLibp2p1.connectionManager.size).to.equal(1) + + // Spy on dial + sinon.spy(relayLibp2p1, 'dial') + + // Remove peer used as relay from peerStore and disconnect it + relayLibp2p1.peerStore.delete(relayLibp2p2.peerId) + await relayLibp2p1.hangUp(relayLibp2p2.peerId) + expect(autoRelay1._listenRelays.size).to.equal(0) + expect(relayLibp2p1.connectionManager.size).to.equal(0) + + // Wait for other peer connected to be added as listen addr + await pWaitFor(() => relayLibp2p1.transportManager.listen.callCount === 2) + expect(autoRelay1._listenRelays.size).to.equal(1) + expect(relayLibp2p1.connectionManager.size).to.eql(1) + }) + }) + + describe('flows with 2 max listeners', () => { + let relayLibp2p1 + let relayLibp2p2 + let relayLibp2p3 + let autoRelay1 + let autoRelay2 + + beforeEach(async () => { + const peerIds = await createPeerId({ number: 3 }) + // Create 3 nodes, and turn HOP on for the relay + ;[relayLibp2p1, relayLibp2p2, relayLibp2p3] = peerIds.map((peerId) => { + return new Libp2p({ + ...baseOptions, + config: { + ...baseOptions.config, + relay: { + ...baseOptions.config.relay, + hop: { + enabled: true + }, + autoRelay: { + enabled: true, + maxListeners: 2 + } + } + }, + addresses: { + listen: [listenAddr] + }, + connectionManager: { + autoDial: false + }, + peerDiscovery: { + autoDial: false + }, + peerId + }) + }) + + autoRelay1 = relayLibp2p1.transportManager._transports.get('Circuit')._autoRelay + autoRelay2 = relayLibp2p2.transportManager._transports.get('Circuit')._autoRelay + }) + + beforeEach(() => { + // Start each node + return Promise.all([relayLibp2p1, relayLibp2p2, relayLibp2p3].map(libp2p => libp2p.start())) + }) + + afterEach(() => { + // Stop each node + return Promise.all([relayLibp2p1, relayLibp2p2, relayLibp2p3].map(libp2p => libp2p.stop())) + }) + + it('should not add listener to a already relayed connection', async () => { + // Spy if a connected peer is being added as listen relay + sinon.spy(autoRelay1, '_addListenRelay') + sinon.spy(autoRelay2, '_addListenRelay') + + // Relay 1 discovers Relay 3 and connect + relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs) + await relayLibp2p1.dial(relayLibp2p3.peerId) + + // Wait for peer added as listen relay + await pWaitFor(() => autoRelay1._addListenRelay.callCount === 1) + expect(autoRelay1._listenRelays.size).to.equal(1) + + // Relay 2 discovers Relay 3 and connect + relayLibp2p2.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs) + await relayLibp2p2.dial(relayLibp2p3.peerId) + + // Wait for peer added as listen relay + await pWaitFor(() => autoRelay2._addListenRelay.callCount === 1) + expect(autoRelay2._listenRelays.size).to.equal(1) + + // Relay 1 discovers Relay 2 relayed multiaddr via Relay 3 + const ma2RelayedBy3 = relayLibp2p2.multiaddrs[relayLibp2p2.multiaddrs.length - 1] + relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, [ma2RelayedBy3]) + await relayLibp2p1.dial(relayLibp2p2.peerId) + + // Peer not added as listen relay + expect(autoRelay1._addListenRelay.callCount).to.equal(1) + expect(autoRelay1._listenRelays.size).to.equal(1) + }) + }) +}) diff --git a/test/dialing/relay.node.js b/test/relay/relay.node.js similarity index 97% rename from test/dialing/relay.node.js rename to test/relay/relay.node.js index a591940801..67f90a7f98 100644 --- a/test/dialing/relay.node.js +++ b/test/relay/relay.node.js @@ -72,7 +72,7 @@ describe('Dialing (via relay, TCP)', () => { const tcpAddrs = dstLibp2p.transportManager.getAddrs() sinon.stub(dstLibp2p.addressManager, 'listen').value([multiaddr(`/p2p-circuit${relayAddr}/p2p/${relayIdString}`)]) - await dstLibp2p.transportManager.listen() + await dstLibp2p.transportManager.listen(dstLibp2p.addressManager.getListenAddrs()) expect(dstLibp2p.transportManager.getAddrs()).to.have.deep.members([...tcpAddrs, dialAddr.decapsulate('p2p')]) const connection = await srcLibp2p.dial(dialAddr) @@ -157,7 +157,7 @@ describe('Dialing (via relay, TCP)', () => { const tcpAddrs = dstLibp2p.transportManager.getAddrs() sinon.stub(dstLibp2p.addressManager, 'getListenAddrs').returns([multiaddr(`${relayAddr}/p2p-circuit`)]) - await dstLibp2p.transportManager.listen() + await dstLibp2p.transportManager.listen(dstLibp2p.addressManager.getListenAddrs()) expect(dstLibp2p.transportManager.getAddrs()).to.have.deep.members([...tcpAddrs, dialAddr.decapsulate('p2p')]) // Tamper with the our multiaddrs for the circuit message diff --git a/test/transports/transport-manager.node.js b/test/transports/transport-manager.node.js index 1036230acb..6e8cc12aea 100644 --- a/test/transports/transport-manager.node.js +++ b/test/transports/transport-manager.node.js @@ -41,7 +41,7 @@ describe('Transport Manager (TCP)', () => { it('should be able to listen', async () => { tm.add(Transport.prototype[Symbol.toStringTag], Transport) - await tm.listen() + await tm.listen(addrs) expect(tm._listeners).to.have.key(Transport.prototype[Symbol.toStringTag]) expect(tm._listeners.get(Transport.prototype[Symbol.toStringTag])).to.have.length(addrs.length) // Ephemeral ip addresses may result in multiple listeners @@ -52,7 +52,7 @@ describe('Transport Manager (TCP)', () => { it('should be able to dial', async () => { tm.add(Transport.prototype[Symbol.toStringTag], Transport) - await tm.listen() + await tm.listen(addrs) const addr = tm.getAddrs().shift() const connection = await tm.dial(addr) expect(connection).to.exist() diff --git a/test/transports/transport-manager.spec.js b/test/transports/transport-manager.spec.js index b32b280725..9f1bbf434c 100644 --- a/test/transports/transport-manager.spec.js +++ b/test/transports/transport-manager.spec.js @@ -87,7 +87,7 @@ describe('Transport Manager (WebSockets)', () => { it('should fail to listen with no valid address', async () => { tm.add(Transport.prototype[Symbol.toStringTag], Transport) - await expect(tm.listen()) + await expect(tm.listen([listenAddr])) .to.eventually.be.rejected() .and.to.have.property('code', ErrorCodes.ERR_NO_VALID_ADDRESSES) }) From 8d75093dcb8ce5fe1e78284ca0199b6491ca80d3 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 10 Sep 2020 11:53:33 +0200 Subject: [PATCH 02/17] chore: auto relay multiaddr update push --- src/circuit/auto-relay.js | 4 ++-- src/circuit/listener.js | 4 ++-- src/identify/index.js | 30 ++++++++++++++++++++++-------- src/index.js | 2 +- test/relay/auto-relay.node.js | 11 ++++++++--- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/circuit/auto-relay.js b/src/circuit/auto-relay.js index 5617e94eff..71e951311b 100644 --- a/src/circuit/auto-relay.js +++ b/src/circuit/auto-relay.js @@ -143,8 +143,8 @@ class AutoRelay { try { await this._transportManager.listen([multiaddr(listenAddr)]) - // TODO: push announce multiaddrs update - // await this._libp2p.identifyService.pushToPeerStore() + // Announce multiaddrs update on listen success + await this._libp2p.identifyService.pushToPeerStore() } catch (err) { log.error(err) this._listenRelays.delete(id) diff --git a/src/circuit/listener.js b/src/circuit/listener.js index f8caff0b41..59ca0cec2c 100644 --- a/src/circuit/listener.js +++ b/src/circuit/listener.js @@ -20,8 +20,8 @@ module.exports = (libp2p) => { const deleted = listeningAddrs.delete(connection.remotePeer.toB58String()) if (deleted) { - // TODO push announce multiaddrs update - // libp2p.identifyService.pushToPeerStore() + // Announce multiaddrs update on listen success + libp2p.identifyService.pushToPeerStore() } }) diff --git a/src/identify/index.js b/src/identify/index.js index ba49610ffc..2340013283 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -319,13 +319,30 @@ class IdentifyService { * @returns {Uint8Array} */ async _getSelfPeerRecord () { - const selfSignedPeerRecord = this.peerStore.addressBook.getRawEnvelope(this.peerId) + // Update self peer record if needed + await this._createOrUpdateSelfPeerRecord() - // TODO: support invalidation when dynamic multiaddrs are supported - if (selfSignedPeerRecord) { - return selfSignedPeerRecord + return this.peerStore.addressBook.getRawEnvelope(this.peerId) + } + + /** + * Creates or updates the self peer record if it exists and is outdated. + * @return {Promise} + */ + async _createOrUpdateSelfPeerRecord () { + const selfPeerRecordEnvelope = await this.peerStore.addressBook.getPeerRecord(this.peerId) + + if (selfPeerRecordEnvelope) { + const peerRecord = PeerRecord.createFromProtobuf(selfPeerRecordEnvelope.payload) + + const mIntersection = peerRecord.multiaddrs.filter((m) => this._libp2p.multiaddrs.some((newM) => m.equals(newM))) + if (mIntersection.length === this._libp2p.multiaddrs.length) { + // Same multiaddrs as already existing in the record, no need to proceed + return + } } + // Create / Update Peer record try { const peerRecord = new PeerRecord({ peerId: this.peerId, @@ -333,12 +350,9 @@ class IdentifyService { }) const envelope = await Envelope.seal(peerRecord, this.peerId) this.peerStore.addressBook.consumePeerRecord(envelope) - - return this.peerStore.addressBook.getRawEnvelope(this.peerId) } catch (err) { - log.error('failed to get self peer record') + log.error('failed to create self peer record') } - return null } } diff --git a/src/index.js b/src/index.js index b269122032..1bcdccf43d 100644 --- a/src/index.js +++ b/src/index.js @@ -259,6 +259,7 @@ class Libp2p extends EventEmitter { await this.peerStore.stop() await this.connectionManager.stop() + ping.unmount(this) await Promise.all([ this.pubsub && this.pubsub.stop(), this._dht && this._dht.stop(), @@ -267,7 +268,6 @@ class Libp2p extends EventEmitter { await this.transportManager.close() - ping.unmount(this) this.dialer.destroy() } catch (err) { if (err) { diff --git a/test/relay/auto-relay.node.js b/test/relay/auto-relay.node.js index 2a4ba20d57..96f94cd7bb 100644 --- a/test/relay/auto-relay.node.js +++ b/test/relay/auto-relay.node.js @@ -259,6 +259,9 @@ describe('auto-relay', () => { it('should not listen on a relayed address if peer disconnects', async () => { const originalMultiaddrs1Length = relayLibp2p1.multiaddrs.length + // Spy if identify push is fired on adding/removing listen addr + sinon.spy(relayLibp2p1.identifyService, 'pushToPeerStore') + // Discover one relay and connect relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs) await relayLibp2p1.dial(relayLibp2p2.peerId) @@ -268,8 +271,8 @@ describe('auto-relay', () => { expect(autoRelay1._listenRelays.size).to.equal(1) expect(relayLibp2p1.multiaddrs[originalMultiaddrs1Length].getPeerId()).to.eql(relayLibp2p2.peerId.toB58String()) - // Spy if identify push is fired - sinon.spy(relayLibp2p1.identifyService, 'pushToPeerStore') + // Identify push for adding listen relay multiaddr + expect(relayLibp2p1.identifyService.pushToPeerStore.callCount).to.equal(1) // Disconnect from peer used for relay await relayLibp2p1.hangUp(relayLibp2p2.peerId) @@ -277,7 +280,9 @@ describe('auto-relay', () => { // Wait for removed listening on the relay await pWaitFor(() => relayLibp2p1.multiaddrs.length === originalMultiaddrs1Length) expect(autoRelay1._listenRelays.size).to.equal(0) - // TODO: identify-push expect(relayLibp2p1.identifyService.pushToPeerStore.callCount).to.equal(1) + + // Identify push for removing listen relay multiaddr + expect(relayLibp2p1.identifyService.pushToPeerStore.callCount).to.equal(2) }) it('should try to listen on other connected peers relayed address if one used relay disconnects', async () => { From 971655ff2718bd0e5773f34b85417ecc4b9a855a Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 23 Sep 2020 11:19:59 +0200 Subject: [PATCH 03/17] chore: _isStarted is false when stop starts --- src/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/index.js b/src/index.js index 1bcdccf43d..4d8807963b 100644 --- a/src/index.js +++ b/src/index.js @@ -248,6 +248,7 @@ class Libp2p extends EventEmitter { log('libp2p is stopping') try { + this._isStarted = false for (const service of this._discovery.values()) { service.removeListener('peer', this._onDiscoveryPeer) } @@ -259,7 +260,6 @@ class Libp2p extends EventEmitter { await this.peerStore.stop() await this.connectionManager.stop() - ping.unmount(this) await Promise.all([ this.pubsub && this.pubsub.stop(), this._dht && this._dht.stop(), @@ -268,6 +268,8 @@ class Libp2p extends EventEmitter { await this.transportManager.close() + ping.unmount(this) + this.dialer.destroy() } catch (err) { if (err) { @@ -275,7 +277,6 @@ class Libp2p extends EventEmitter { this.emit('error', err) } } - this._isStarted = false log('libp2p has stopped') } From ee8ee5b49b77a2ac93a8cc824a50939712a4827a Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 23 Sep 2020 13:14:53 +0200 Subject: [PATCH 04/17] chore: use listening events to create self peer record on updates --- doc/API.md | 9 ++++++ src/circuit/auto-relay.js | 3 +- src/circuit/listener.js | 4 +-- src/identify/index.js | 59 +++++++++++++++++++------------------ src/index.js | 1 - src/transport-manager.js | 14 ++++++++- test/identify/index.spec.js | 10 +++++++ 7 files changed, 66 insertions(+), 34 deletions(-) diff --git a/doc/API.md b/doc/API.md index 69e0ca4f78..636d57bb15 100644 --- a/doc/API.md +++ b/doc/API.md @@ -73,6 +73,7 @@ * [`libp2p`](#libp2p) * [`libp2p.connectionManager`](#libp2pconnectionmanager) * [`libp2p.peerStore`](#libp2ppeerStore) + * [`libp2p.transportManager`](#libp2ptransportmanager) * [Types](#types) * [`Stats`](#stats) @@ -1987,6 +1988,14 @@ This event will be triggered anytime we are disconnected from another peer, rega - `peerId`: instance of [`PeerId`][peer-id] - `protocols`: array of known, supported protocols for the peer (string identifiers) +### libp2p.transportManager + +#### Listening addresses change + +This event will be triggered anytime the listening addresses change. + +`libp2p.transportManager.on('listening', () => {})` + ## Types ### Stats diff --git a/src/circuit/auto-relay.js b/src/circuit/auto-relay.js index 71e951311b..bec8c3a8f0 100644 --- a/src/circuit/auto-relay.js +++ b/src/circuit/auto-relay.js @@ -143,8 +143,7 @@ class AutoRelay { try { await this._transportManager.listen([multiaddr(listenAddr)]) - // Announce multiaddrs update on listen success - await this._libp2p.identifyService.pushToPeerStore() + // Announce multiaddrs will update on listen success by TransportManager event being triggered } catch (err) { log.error(err) this._listenRelays.delete(id) diff --git a/src/circuit/listener.js b/src/circuit/listener.js index 59ca0cec2c..207150e8a8 100644 --- a/src/circuit/listener.js +++ b/src/circuit/listener.js @@ -20,8 +20,8 @@ module.exports = (libp2p) => { const deleted = listeningAddrs.delete(connection.remotePeer.toB58String()) if (deleted) { - // Announce multiaddrs update on listen success - libp2p.identifyService.pushToPeerStore() + // Announce listen addresses change + listener.emit('listening') } }) diff --git a/src/identify/index.js b/src/identify/index.js index 2340013283..dad7e4968c 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -64,11 +64,10 @@ class IdentifyService { */ this.connectionManager = libp2p.connectionManager - this.connectionManager.on('peer:connect', (connection) => { - const peerId = connection.remotePeer - - this.identify(connection, peerId).catch(log.error) - }) + /** + * @property {TransportManager} + */ + this.transportManager = libp2p.transportManager /** * @property {PeerId} @@ -83,6 +82,18 @@ class IdentifyService { this._protocols = protocols this.handleMessage = this.handleMessage.bind(this) + + this.connectionManager.on('peer:connect', (connection) => { + const peerId = connection.remotePeer + + this.identify(connection, peerId).catch(log.error) + }) + + // When new addresses are used for listening, update self peer record + this.transportManager.on('listening', async () => { + await this._createSelfPeerRecord() + this.pushToPeerStore() + }) } /** @@ -315,34 +326,23 @@ class IdentifyService { /** * Get self signed peer record raw envelope. - * - * @returns {Uint8Array} + * @return {Promise} */ - async _getSelfPeerRecord () { - // Update self peer record if needed - await this._createOrUpdateSelfPeerRecord() + _getSelfPeerRecord () { + const selfSignedPeerRecord = this.peerStore.addressBook.getRawEnvelope(this.peerId) - return this.peerStore.addressBook.getRawEnvelope(this.peerId) + if (selfSignedPeerRecord) { + return selfSignedPeerRecord + } + + return this._createSelfPeerRecord() } /** - * Creates or updates the self peer record if it exists and is outdated. - * @return {Promise} + * Create self signed peer record raw envelope. + * @return {Uint8Array} */ - async _createOrUpdateSelfPeerRecord () { - const selfPeerRecordEnvelope = await this.peerStore.addressBook.getPeerRecord(this.peerId) - - if (selfPeerRecordEnvelope) { - const peerRecord = PeerRecord.createFromProtobuf(selfPeerRecordEnvelope.payload) - - const mIntersection = peerRecord.multiaddrs.filter((m) => this._libp2p.multiaddrs.some((newM) => m.equals(newM))) - if (mIntersection.length === this._libp2p.multiaddrs.length) { - // Same multiaddrs as already existing in the record, no need to proceed - return - } - } - - // Create / Update Peer record + async _createSelfPeerRecord () { try { const peerRecord = new PeerRecord({ peerId: this.peerId, @@ -350,9 +350,12 @@ class IdentifyService { }) const envelope = await Envelope.seal(peerRecord, this.peerId) this.peerStore.addressBook.consumePeerRecord(envelope) + + return this.peerStore.addressBook.getRawEnvelope(this.peerId) } catch (err) { - log.error('failed to create self peer record') + log.error('failed to get self peer record') } + return null } } diff --git a/src/index.js b/src/index.js index 4d8807963b..2e46ebd5d7 100644 --- a/src/index.js +++ b/src/index.js @@ -269,7 +269,6 @@ class Libp2p extends EventEmitter { await this.transportManager.close() ping.unmount(this) - this.dialer.destroy() } catch (err) { if (err) { diff --git a/src/transport-manager.js b/src/transport-manager.js index 7570389073..8c743fe092 100644 --- a/src/transport-manager.js +++ b/src/transport-manager.js @@ -1,5 +1,6 @@ 'use strict' +const { EventEmitter } = require('events') const pSettle = require('p-settle') const { codes } = require('./errors') const errCode = require('err-code') @@ -7,7 +8,11 @@ const debug = require('debug') const log = debug('libp2p:transports') log.error = debug('libp2p:transports:error') -class TransportManager { +/** + * Responsible for managing the transports and their listeners. + * @fires TransportManager#listening Emitted when listening addresses change. + */ +class TransportManager extends EventEmitter { /** * @class * @param {object} options @@ -16,6 +21,8 @@ class TransportManager { * @param {boolean} [options.faultTolerance = FAULT_TOLERANCE.FATAL_ALL] - Address listen error tolerance. */ constructor ({ libp2p, upgrader, faultTolerance = FAULT_TOLERANCE.FATAL_ALL }) { + super() + this.libp2p = libp2p this.upgrader = upgrader this._transports = new Map() @@ -63,6 +70,7 @@ class TransportManager { log('closing listeners for %s', key) while (listeners.length) { const listener = listeners.pop() + listener.removeAllListeners('listening') tasks.push(listener.close()) } } @@ -156,6 +164,9 @@ class TransportManager { const listener = transport.createListener({}, this.onConnection) this._listeners.get(key).push(listener) + // Track listen events + listener.on('listening', () => this.emit('listening')) + // We need to attempt to listen on everything tasks.push(listener.listen(addr)) } @@ -200,6 +211,7 @@ class TransportManager { if (this._listeners.has(key)) { // Close any running listeners for (const listener of this._listeners.get(key)) { + listener.removeAllListeners('listening') await listener.close() } } diff --git a/test/identify/index.spec.js b/test/identify/index.spec.js index 1ccbf67122..9443e2f5ba 100644 --- a/test/identify/index.spec.js +++ b/test/identify/index.spec.js @@ -52,6 +52,7 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), + transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: localPeer }), multiaddrs: listenMaddrs }, @@ -62,6 +63,7 @@ describe('Identify', () => { libp2p: { peerId: remotePeer, connectionManager: new EventEmitter(), + transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: remotePeer }), multiaddrs: listenMaddrs }, @@ -105,6 +107,7 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), + transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: localPeer }), multiaddrs: listenMaddrs }, @@ -115,6 +118,7 @@ describe('Identify', () => { libp2p: { peerId: remotePeer, connectionManager: new EventEmitter(), + transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: remotePeer }), multiaddrs: listenMaddrs }, @@ -164,6 +168,7 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), + transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: localPeer }), multiaddrs: [] }, @@ -173,6 +178,7 @@ describe('Identify', () => { libp2p: { peerId: remotePeer, connectionManager: new EventEmitter(), + transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: remotePeer }), multiaddrs: [] }, @@ -210,6 +216,7 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), + transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: localPeer }), multiaddrs: listenMaddrs }, @@ -223,6 +230,7 @@ describe('Identify', () => { libp2p: { peerId: remotePeer, connectionManager, + transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: remotePeer }), multiaddrs: [] } @@ -271,6 +279,7 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), + transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: localPeer }), multiaddrs: listenMaddrs }, @@ -284,6 +293,7 @@ describe('Identify', () => { libp2p: { peerId: remotePeer, connectionManager, + transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: remotePeer }), multiaddrs: [] } From 87d20ac46dbe242455ba8ccbc81ee99109dca485 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 23 Sep 2020 18:45:01 +0200 Subject: [PATCH 05/17] chore: create signed peer record on new listen addresses in transport manager --- doc/API.md | 9 ---- src/circuit/listener.js | 2 +- src/identify/index.js | 52 +++-------------------- src/transport-manager.js | 39 ++++++++++++----- test/dialing/direct.spec.js | 3 -- test/identify/index.spec.js | 39 +++++++++++------ test/transports/transport-manager.node.js | 18 +++++++- 7 files changed, 80 insertions(+), 82 deletions(-) diff --git a/doc/API.md b/doc/API.md index 636d57bb15..69e0ca4f78 100644 --- a/doc/API.md +++ b/doc/API.md @@ -73,7 +73,6 @@ * [`libp2p`](#libp2p) * [`libp2p.connectionManager`](#libp2pconnectionmanager) * [`libp2p.peerStore`](#libp2ppeerStore) - * [`libp2p.transportManager`](#libp2ptransportmanager) * [Types](#types) * [`Stats`](#stats) @@ -1988,14 +1987,6 @@ This event will be triggered anytime we are disconnected from another peer, rega - `peerId`: instance of [`PeerId`][peer-id] - `protocols`: array of known, supported protocols for the peer (string identifiers) -### libp2p.transportManager - -#### Listening addresses change - -This event will be triggered anytime the listening addresses change. - -`libp2p.transportManager.on('listening', () => {})` - ## Types ### Stats diff --git a/src/circuit/listener.js b/src/circuit/listener.js index 207150e8a8..02e371fb8b 100644 --- a/src/circuit/listener.js +++ b/src/circuit/listener.js @@ -21,7 +21,7 @@ module.exports = (libp2p) => { if (deleted) { // Announce listen addresses change - listener.emit('listening') + listener.emit('close') } }) diff --git a/src/identify/index.js b/src/identify/index.js index dad7e4968c..d7202ed9c4 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -64,11 +64,6 @@ class IdentifyService { */ this.connectionManager = libp2p.connectionManager - /** - * @property {TransportManager} - */ - this.transportManager = libp2p.transportManager - /** * @property {PeerId} */ @@ -89,10 +84,11 @@ class IdentifyService { this.identify(connection, peerId).catch(log.error) }) - // When new addresses are used for listening, update self peer record - this.transportManager.on('listening', async () => { - await this._createSelfPeerRecord() - this.pushToPeerStore() + // When self multiaddrs change, trigger identify-push + this.peerStore.on('change:multiaddrs', ({ peerId }) => { + if (peerId.toString() === this.peerId.toString()) { + this.pushToPeerStore() + } }) } @@ -103,7 +99,7 @@ class IdentifyService { * @returns {Promise} */ async push (connections) { - const signedPeerRecord = await this._getSelfPeerRecord() + const signedPeerRecord = await this.peerStore.addressBook.getRawEnvelope(this.peerId) const listenAddrs = this._libp2p.multiaddrs.map((ma) => ma.bytes) const protocols = Array.from(this._protocols.keys()) @@ -253,7 +249,7 @@ class IdentifyService { publicKey = this.peerId.pubKey.bytes } - const signedPeerRecord = await this._getSelfPeerRecord() + const signedPeerRecord = await this.peerStore.addressBook.getRawEnvelope(this.peerId) const message = Message.encode({ protocolVersion: PROTOCOL_VERSION, @@ -323,40 +319,6 @@ class IdentifyService { // Update the protocols this.peerStore.protoBook.set(id, message.protocols) } - - /** - * Get self signed peer record raw envelope. - * @return {Promise} - */ - _getSelfPeerRecord () { - const selfSignedPeerRecord = this.peerStore.addressBook.getRawEnvelope(this.peerId) - - if (selfSignedPeerRecord) { - return selfSignedPeerRecord - } - - return this._createSelfPeerRecord() - } - - /** - * Create self signed peer record raw envelope. - * @return {Uint8Array} - */ - async _createSelfPeerRecord () { - try { - const peerRecord = new PeerRecord({ - peerId: this.peerId, - multiaddrs: this._libp2p.multiaddrs - }) - const envelope = await Envelope.seal(peerRecord, this.peerId) - this.peerStore.addressBook.consumePeerRecord(envelope) - - return this.peerStore.addressBook.getRawEnvelope(this.peerId) - } catch (err) { - log.error('failed to get self peer record') - } - return null - } } module.exports.IdentifyService = IdentifyService diff --git a/src/transport-manager.js b/src/transport-manager.js index 8c743fe092..08256d9aab 100644 --- a/src/transport-manager.js +++ b/src/transport-manager.js @@ -1,6 +1,5 @@ 'use strict' -const { EventEmitter } = require('events') const pSettle = require('p-settle') const { codes } = require('./errors') const errCode = require('err-code') @@ -8,11 +7,10 @@ const debug = require('debug') const log = debug('libp2p:transports') log.error = debug('libp2p:transports:error') -/** - * Responsible for managing the transports and their listeners. - * @fires TransportManager#listening Emitted when listening addresses change. - */ -class TransportManager extends EventEmitter { +const Envelope = require('./record/envelope') +const PeerRecord = require('./record/peer-record') + +class TransportManager { /** * @class * @param {object} options @@ -21,8 +19,6 @@ class TransportManager extends EventEmitter { * @param {boolean} [options.faultTolerance = FAULT_TOLERANCE.FATAL_ALL] - Address listen error tolerance. */ constructor ({ libp2p, upgrader, faultTolerance = FAULT_TOLERANCE.FATAL_ALL }) { - super() - this.libp2p = libp2p this.upgrader = upgrader this._transports = new Map() @@ -71,6 +67,7 @@ class TransportManager extends EventEmitter { while (listeners.length) { const listener = listeners.pop() listener.removeAllListeners('listening') + listener.removeAllListeners('close') tasks.push(listener.close()) } } @@ -164,8 +161,9 @@ class TransportManager extends EventEmitter { const listener = transport.createListener({}, this.onConnection) this._listeners.get(key).push(listener) - // Track listen events - listener.on('listening', () => this.emit('listening')) + // Track listen/close events + listener.on('listening', () => this._createSelfPeerRecord()) + listener.on('close', () => this._createSelfPeerRecord()) // We need to attempt to listen on everything tasks.push(listener.listen(addr)) @@ -212,6 +210,7 @@ class TransportManager extends EventEmitter { // Close any running listeners for (const listener of this._listeners.get(key)) { listener.removeAllListeners('listening') + listener.removeAllListeners('close') await listener.close() } } @@ -234,6 +233,26 @@ class TransportManager extends EventEmitter { await Promise.all(tasks) } + + /** + * Create self signed peer record raw envelope. + * @return {Uint8Array} + */ + async _createSelfPeerRecord () { + try { + const peerRecord = new PeerRecord({ + peerId: this.libp2p.peerId, + multiaddrs: this.libp2p.multiaddrs + }) + const envelope = await Envelope.seal(peerRecord, this.libp2p.peerId) + this.libp2p.peerStore.addressBook.consumePeerRecord(envelope) + + return this.libp2p.peerStore.addressBook.getRawEnvelope(this.libp2p.peerId) + } catch (err) { + log.error('failed to get self peer record') + } + return null + } } /** diff --git a/test/dialing/direct.spec.js b/test/dialing/direct.spec.js index 540f528b65..0659ed9aa3 100644 --- a/test/dialing/direct.spec.js +++ b/test/dialing/direct.spec.js @@ -349,7 +349,6 @@ describe('Dialing (direct, WebSockets)', () => { const connection = await libp2p.dial(remoteAddr) expect(connection).to.exist() - sinon.spy(libp2p.peerStore.addressBook, 'consumePeerRecord') sinon.spy(libp2p.peerStore.protoBook, 'set') // Wait for onConnection to be called @@ -358,8 +357,6 @@ describe('Dialing (direct, WebSockets)', () => { expect(libp2p.identifyService.identify.callCount).to.equal(1) await libp2p.identifyService.identify.firstCall.returnValue - // Self + New peer - expect(libp2p.peerStore.addressBook.consumePeerRecord.callCount).to.equal(2) expect(libp2p.peerStore.protoBook.set.callCount).to.equal(1) }) diff --git a/test/identify/index.spec.js b/test/identify/index.spec.js index 9443e2f5ba..9abbe9bee5 100644 --- a/test/identify/index.spec.js +++ b/test/identify/index.spec.js @@ -20,6 +20,7 @@ const { IdentifyService, multicodecs } = require('../../src/identify') const Peers = require('../fixtures/peers') const Libp2p = require('../../src') const Envelope = require('../../src/record/envelope') +const PeerRecord = require('../../src/record/peer-record') const PeerStore = require('../../src/peer-store') const baseOptions = require('../utils/base-options.browser') const pkg = require('../../package.json') @@ -52,7 +53,6 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), - transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: localPeer }), multiaddrs: listenMaddrs }, @@ -63,7 +63,6 @@ describe('Identify', () => { libp2p: { peerId: remotePeer, connectionManager: new EventEmitter(), - transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: remotePeer }), multiaddrs: listenMaddrs }, @@ -80,6 +79,9 @@ describe('Identify', () => { sinon.spy(localIdentify.peerStore.addressBook, 'consumePeerRecord') sinon.spy(localIdentify.peerStore.protoBook, 'set') + // Transport Manager creates signed peer record + await _createSelfPeerRecord(remoteIdentify._libp2p) + // Run identify await Promise.all([ localIdentify.identify(localConnectionMock), @@ -107,7 +109,6 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), - transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: localPeer }), multiaddrs: listenMaddrs }, @@ -118,7 +119,6 @@ describe('Identify', () => { libp2p: { peerId: remotePeer, connectionManager: new EventEmitter(), - transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: remotePeer }), multiaddrs: listenMaddrs }, @@ -168,7 +168,6 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), - transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: localPeer }), multiaddrs: [] }, @@ -178,7 +177,6 @@ describe('Identify', () => { libp2p: { peerId: remotePeer, connectionManager: new EventEmitter(), - transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: remotePeer }), multiaddrs: [] }, @@ -216,7 +214,6 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), - transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: localPeer }), multiaddrs: listenMaddrs }, @@ -230,7 +227,6 @@ describe('Identify', () => { libp2p: { peerId: remotePeer, connectionManager, - transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: remotePeer }), multiaddrs: [] } @@ -247,6 +243,10 @@ describe('Identify', () => { sinon.spy(remoteIdentify.peerStore.addressBook, 'consumePeerRecord') sinon.spy(remoteIdentify.peerStore.protoBook, 'set') + // Transport Manager creates signed peer record + await _createSelfPeerRecord(localIdentify._libp2p) + await _createSelfPeerRecord(remoteIdentify._libp2p) + // Run identify await Promise.all([ localIdentify.push([localConnectionMock]), @@ -257,7 +257,7 @@ describe('Identify', () => { }) ]) - expect(remoteIdentify.peerStore.addressBook.consumePeerRecord.callCount).to.equal(1) + expect(remoteIdentify.peerStore.addressBook.consumePeerRecord.callCount).to.equal(2) expect(remoteIdentify.peerStore.protoBook.set.callCount).to.equal(1) const addresses = localIdentify.peerStore.addressBook.get(localPeer) @@ -279,7 +279,6 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), - transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: localPeer }), multiaddrs: listenMaddrs }, @@ -293,7 +292,6 @@ describe('Identify', () => { libp2p: { peerId: remotePeer, connectionManager, - transportManager: new EventEmitter(), peerStore: new PeerStore({ peerId: remotePeer }), multiaddrs: [] } @@ -369,8 +367,8 @@ describe('Identify', () => { expect(connection).to.exist() // Wait for peer store to be updated - // Dialer._createDialTarget (add), Identify (consume), Create self (consume) - await pWaitFor(() => peerStoreSpyConsumeRecord.callCount === 2 && peerStoreSpyAdd.callCount === 1) + // Dialer._createDialTarget (add), Identify (consume) + await pWaitFor(() => peerStoreSpyConsumeRecord.callCount === 1 && peerStoreSpyAdd.callCount === 1) expect(libp2p.identifyService.identify.callCount).to.equal(1) // The connection should have no open streams @@ -416,3 +414,18 @@ describe('Identify', () => { }) }) }) + +// Self peer record creating on Transport Manager simulation +const _createSelfPeerRecord = async (libp2p) => { + try { + const peerRecord = new PeerRecord({ + peerId: libp2p.peerId, + multiaddrs: libp2p.multiaddrs + }) + const envelope = await Envelope.seal(peerRecord, libp2p.peerId) + libp2p.peerStore.addressBook.consumePeerRecord(envelope) + + return libp2p.peerStore.addressBook.getRawEnvelope(libp2p.peerId) + } catch (_) {} + return null +} diff --git a/test/transports/transport-manager.node.js b/test/transports/transport-manager.node.js index 6e8cc12aea..f8a1c633af 100644 --- a/test/transports/transport-manager.node.js +++ b/test/transports/transport-manager.node.js @@ -4,12 +4,16 @@ const chai = require('chai') chai.use(require('dirty-chai')) const { expect } = chai +const sinon = require('sinon') const AddressManager = require('../../src/address-manager') const TransportManager = require('../../src/transport-manager') +const PeerStore = require('../../src/peer-store') const Transport = require('libp2p-tcp') +const PeerId = require('peer-id') const multiaddr = require('multiaddr') const mockUpgrader = require('../utils/mockUpgrader') +const Peers = require('../fixtures/peers') const addrs = [ multiaddr('/ip4/127.0.0.1/tcp/0'), multiaddr('/ip4/127.0.0.1/tcp/0') @@ -17,11 +21,17 @@ const addrs = [ describe('Transport Manager (TCP)', () => { let tm + let localPeer + + before(async () => { + localPeer = await PeerId.createFromJSON(Peers[0]) + }) before(() => { tm = new TransportManager({ libp2p: { - addressManager: new AddressManager({ listen: addrs }) + addressManager: new AddressManager({ listen: addrs }), + PeerStore: new PeerStore({ peerId: localPeer }) }, upgrader: mockUpgrader, onConnection: () => {} @@ -40,10 +50,16 @@ describe('Transport Manager (TCP)', () => { }) it('should be able to listen', async () => { + sinon.spy(tm, '_createSelfPeerRecord') + tm.add(Transport.prototype[Symbol.toStringTag], Transport) await tm.listen(addrs) expect(tm._listeners).to.have.key(Transport.prototype[Symbol.toStringTag]) expect(tm._listeners.get(Transport.prototype[Symbol.toStringTag])).to.have.length(addrs.length) + + // Created Self Peer record on new listen address + expect(tm._createSelfPeerRecord.callCount).to.equal(addrs.length) + // Ephemeral ip addresses may result in multiple listeners expect(tm.getAddrs().length).to.equal(addrs.length) await tm.close() From abba305bd6ec096a86c189b90db40b00ea514ad7 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 24 Sep 2020 11:12:02 +0200 Subject: [PATCH 06/17] chore: add identify test for multiaddr change --- test/identify/index.spec.js | 39 ++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/test/identify/index.spec.js b/test/identify/index.spec.js index 9abbe9bee5..72e46fd927 100644 --- a/test/identify/index.spec.js +++ b/test/identify/index.spec.js @@ -412,6 +412,42 @@ describe('Identify', () => { // Verify the streams close await pWaitFor(() => connection.streams.length === 0) }) + + it('should push multiaddr updates to an already connected peer', async () => { + libp2p = new Libp2p({ + ...baseOptions, + peerId + }) + + await libp2p.start() + + sinon.spy(libp2p.identifyService, 'identify') + sinon.spy(libp2p.identifyService, 'push') + + const connection = await libp2p.dialer.connectToPeer(remoteAddr) + expect(connection).to.exist() + // Wait for nextTick to trigger the identify call + await delay(1) + + // Wait for identify to finish + await libp2p.identifyService.identify.firstCall.returnValue + sinon.stub(libp2p, 'isStarted').returns(true) + + libp2p.peerStore.addressBook.add(libp2p.peerId, [multiaddr('/ip4/180.0.0.1/tcp/15001/ws')]) + + // Verify the remote peer is notified of change + expect(libp2p.identifyService.push.callCount).to.equal(1) + for (const call of libp2p.identifyService.push.getCalls()) { + const [connections] = call.args + expect(connections.length).to.equal(1) + expect(connections[0].remotePeer.toB58String()).to.equal(remoteAddr.getPeerId()) + const results = await call.returnValue + expect(results.length).to.equal(1) + } + + // Verify the streams close + await pWaitFor(() => connection.streams.length === 0) + }) }) }) @@ -424,8 +460,5 @@ const _createSelfPeerRecord = async (libp2p) => { }) const envelope = await Envelope.seal(peerRecord, libp2p.peerId) libp2p.peerStore.addressBook.consumePeerRecord(envelope) - - return libp2p.peerStore.addressBook.getRawEnvelope(libp2p.peerId) } catch (_) {} - return null } From 05e6472cce26f6c152407a61a15a35c9a9b89468 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 24 Sep 2020 12:49:48 +0200 Subject: [PATCH 07/17] chore: address review --- src/record/utils.js | 20 ++++++++++++++ src/transport-manager.js | 27 +++---------------- test/dialing/direct.node.js | 23 ++++++++++------ test/identify/index.spec.js | 25 +++--------------- test/transports/transport-manager.node.js | 32 +++++++++++++++++------ 5 files changed, 66 insertions(+), 61 deletions(-) create mode 100644 src/record/utils.js diff --git a/src/record/utils.js b/src/record/utils.js new file mode 100644 index 0000000000..509fea7eec --- /dev/null +++ b/src/record/utils.js @@ -0,0 +1,20 @@ +'use strict' + +const Envelope = require('./envelope') +const PeerRecord = require('./peer-record') + +/** + * Create (or update if existing) self peer record and store it in the AddressBook. + * @param {libp2p} libp2p + * @returns {Promise} + */ +async function updateSelfPeerRecord (libp2p) { + const peerRecord = new PeerRecord({ + peerId: libp2p.peerId, + multiaddrs: libp2p.multiaddrs + }) + const envelope = await Envelope.seal(peerRecord, libp2p.peerId) + libp2p.peerStore.addressBook.consumePeerRecord(envelope) +} + +module.exports.updateSelfPeerRecord = updateSelfPeerRecord diff --git a/src/transport-manager.js b/src/transport-manager.js index 08256d9aab..039d069fd8 100644 --- a/src/transport-manager.js +++ b/src/transport-manager.js @@ -7,8 +7,7 @@ const debug = require('debug') const log = debug('libp2p:transports') log.error = debug('libp2p:transports:error') -const Envelope = require('./record/envelope') -const PeerRecord = require('./record/peer-record') +const { updateSelfPeerRecord } = require('./record/utils') class TransportManager { /** @@ -162,8 +161,8 @@ class TransportManager { this._listeners.get(key).push(listener) // Track listen/close events - listener.on('listening', () => this._createSelfPeerRecord()) - listener.on('close', () => this._createSelfPeerRecord()) + listener.on('listening', () => updateSelfPeerRecord(this.libp2p)) + listener.on('close', () => updateSelfPeerRecord(this.libp2p)) // We need to attempt to listen on everything tasks.push(listener.listen(addr)) @@ -233,26 +232,6 @@ class TransportManager { await Promise.all(tasks) } - - /** - * Create self signed peer record raw envelope. - * @return {Uint8Array} - */ - async _createSelfPeerRecord () { - try { - const peerRecord = new PeerRecord({ - peerId: this.libp2p.peerId, - multiaddrs: this.libp2p.multiaddrs - }) - const envelope = await Envelope.seal(peerRecord, this.libp2p.peerId) - this.libp2p.peerStore.addressBook.consumePeerRecord(envelope) - - return this.libp2p.peerStore.addressBook.getRawEnvelope(this.libp2p.peerId) - } catch (err) { - log.error('failed to get self peer record') - } - return null - } } /** diff --git a/test/dialing/direct.node.js b/test/dialing/direct.node.js index 6b89fee4af..0d9d1dd718 100644 --- a/test/dialing/direct.node.js +++ b/test/dialing/direct.node.js @@ -42,21 +42,28 @@ describe('Dialing (direct, TCP)', () => { let peerStore let remoteAddr - before(async () => { - const [remotePeerId] = await Promise.all([ - PeerId.createFromJSON(Peers[0]) + beforeEach(async () => { + const [localPeerId, remotePeerId] = await Promise.all([ + PeerId.createFromJSON(Peers[0]), + PeerId.createFromJSON(Peers[1]) ]) + + peerStore = new PeerStore({ peerId: remotePeerId }) remoteTM = new TransportManager({ libp2p: { - addressManager: new AddressManager({ listen: [listenAddr] }) + addressManager: new AddressManager({ listen: [listenAddr] }), + peerId: remotePeerId, + peerStore }, upgrader: mockUpgrader }) remoteTM.add(Transport.prototype[Symbol.toStringTag], Transport) - peerStore = new PeerStore({ peerId: remotePeerId }) localTM = new TransportManager({ - libp2p: {}, + libp2p: { + peerId: localPeerId, + peerStore: new PeerStore({ peerId: localPeerId }) + }, upgrader: mockUpgrader }) localTM.add(Transport.prototype[Symbol.toStringTag], Transport) @@ -66,7 +73,7 @@ describe('Dialing (direct, TCP)', () => { remoteAddr = remoteTM.getAddrs()[0].encapsulate(`/p2p/${remotePeerId.toB58String()}`) }) - after(() => remoteTM.close()) + afterEach(() => remoteTM.close()) afterEach(() => { sinon.restore() @@ -112,7 +119,7 @@ describe('Dialing (direct, TCP)', () => { peerStore }) - peerStore.addressBook.set(peerId, [remoteAddr]) + peerStore.addressBook.set(peerId, remoteTM.getAddrs()) const connection = await dialer.connectToPeer(peerId) expect(connection).to.exist() diff --git a/test/identify/index.spec.js b/test/identify/index.spec.js index 72e46fd927..d3b14273b7 100644 --- a/test/identify/index.spec.js +++ b/test/identify/index.spec.js @@ -8,7 +8,6 @@ const { expect } = chai const sinon = require('sinon') const { EventEmitter } = require('events') -const delay = require('delay') const PeerId = require('peer-id') const duplexPair = require('it-pair/duplex') const multiaddr = require('multiaddr') @@ -20,9 +19,9 @@ const { IdentifyService, multicodecs } = require('../../src/identify') const Peers = require('../fixtures/peers') const Libp2p = require('../../src') const Envelope = require('../../src/record/envelope') -const PeerRecord = require('../../src/record/peer-record') const PeerStore = require('../../src/peer-store') const baseOptions = require('../utils/base-options.browser') +const { updateSelfPeerRecord } = require('../../src/record/utils') const pkg = require('../../package.json') const { MULTIADDRS_WEBSOCKETS } = require('../fixtures/browser') @@ -80,7 +79,7 @@ describe('Identify', () => { sinon.spy(localIdentify.peerStore.protoBook, 'set') // Transport Manager creates signed peer record - await _createSelfPeerRecord(remoteIdentify._libp2p) + await updateSelfPeerRecord(remoteIdentify._libp2p) // Run identify await Promise.all([ @@ -244,8 +243,8 @@ describe('Identify', () => { sinon.spy(remoteIdentify.peerStore.protoBook, 'set') // Transport Manager creates signed peer record - await _createSelfPeerRecord(localIdentify._libp2p) - await _createSelfPeerRecord(remoteIdentify._libp2p) + await updateSelfPeerRecord(localIdentify._libp2p) + await updateSelfPeerRecord(remoteIdentify._libp2p) // Run identify await Promise.all([ @@ -389,8 +388,6 @@ describe('Identify', () => { const connection = await libp2p.dialer.connectToPeer(remoteAddr) expect(connection).to.exist() - // Wait for nextTick to trigger the identify call - await delay(1) // Wait for identify to finish await libp2p.identifyService.identify.firstCall.returnValue @@ -426,8 +423,6 @@ describe('Identify', () => { const connection = await libp2p.dialer.connectToPeer(remoteAddr) expect(connection).to.exist() - // Wait for nextTick to trigger the identify call - await delay(1) // Wait for identify to finish await libp2p.identifyService.identify.firstCall.returnValue @@ -450,15 +445,3 @@ describe('Identify', () => { }) }) }) - -// Self peer record creating on Transport Manager simulation -const _createSelfPeerRecord = async (libp2p) => { - try { - const peerRecord = new PeerRecord({ - peerId: libp2p.peerId, - multiaddrs: libp2p.multiaddrs - }) - const envelope = await Envelope.seal(peerRecord, libp2p.peerId) - libp2p.peerStore.addressBook.consumePeerRecord(envelope) - } catch (_) {} -} diff --git a/test/transports/transport-manager.node.js b/test/transports/transport-manager.node.js index f8a1c633af..e123c5a370 100644 --- a/test/transports/transport-manager.node.js +++ b/test/transports/transport-manager.node.js @@ -4,11 +4,11 @@ const chai = require('chai') chai.use(require('dirty-chai')) const { expect } = chai -const sinon = require('sinon') const AddressManager = require('../../src/address-manager') const TransportManager = require('../../src/transport-manager') const PeerStore = require('../../src/peer-store') +const PeerRecord = require('../../src/record/peer-record') const Transport = require('libp2p-tcp') const PeerId = require('peer-id') const multiaddr = require('multiaddr') @@ -27,11 +27,13 @@ describe('Transport Manager (TCP)', () => { localPeer = await PeerId.createFromJSON(Peers[0]) }) - before(() => { + beforeEach(() => { tm = new TransportManager({ libp2p: { + peerId: localPeer, + multiaddrs: addrs, addressManager: new AddressManager({ listen: addrs }), - PeerStore: new PeerStore({ peerId: localPeer }) + peerStore: new PeerStore({ peerId: localPeer }) }, upgrader: mockUpgrader, onConnection: () => {} @@ -50,22 +52,36 @@ describe('Transport Manager (TCP)', () => { }) it('should be able to listen', async () => { - sinon.spy(tm, '_createSelfPeerRecord') - tm.add(Transport.prototype[Symbol.toStringTag], Transport) await tm.listen(addrs) expect(tm._listeners).to.have.key(Transport.prototype[Symbol.toStringTag]) expect(tm._listeners.get(Transport.prototype[Symbol.toStringTag])).to.have.length(addrs.length) - // Created Self Peer record on new listen address - expect(tm._createSelfPeerRecord.callCount).to.equal(addrs.length) - // Ephemeral ip addresses may result in multiple listeners expect(tm.getAddrs().length).to.equal(addrs.length) await tm.close() expect(tm._listeners.get(Transport.prototype[Symbol.toStringTag])).to.have.length(0) }) + it('should create self signed peer record on listen', async () => { + let signedPeerRecord = await tm.libp2p.peerStore.addressBook.getPeerRecord(localPeer) + expect(signedPeerRecord).to.not.exist() + + tm.add(Transport.prototype[Symbol.toStringTag], Transport) + await tm.listen(addrs) + + // Should created Self Peer record on new listen address + signedPeerRecord = await tm.libp2p.peerStore.addressBook.getPeerRecord(localPeer) + expect(signedPeerRecord).to.exist() + + const record = PeerRecord.createFromProtobuf(signedPeerRecord.payload) + expect(record).to.exist() + expect(record.multiaddrs.length).to.equal(addrs.length) + addrs.forEach((a, i) => { + expect(record.multiaddrs[i].equals(a)).to.be.true() + }) + }) + it('should be able to dial', async () => { tm.add(Transport.prototype[Symbol.toStringTag], Transport) await tm.listen(addrs) From 2530b834a13057c31c75a060dced640f07312ccf Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 7 Oct 2020 17:29:42 +0200 Subject: [PATCH 08/17] chore: lint issue fixed 0.30 --- src/circuit/auto-relay.js | 28 +++++++++++++++++----------- src/circuit/circuit/hop.js | 3 ++- src/identify/index.js | 1 + src/record/utils.js | 1 + src/transport-manager.js | 2 +- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/circuit/auto-relay.js b/src/circuit/auto-relay.js index bec8c3a8f0..c0dafc4b17 100644 --- a/src/circuit/auto-relay.js +++ b/src/circuit/auto-relay.js @@ -19,10 +19,11 @@ const hopMetadataValue = 'true' class AutoRelay { /** * Creates an instance of AutoRelay. - * @constructor + * + * @class * @param {object} props * @param {Libp2p} props.libp2p - * @param {number} [props.maxListeners = 1] maximum number of relays to listen. + * @param {number} [props.maxListeners = 1] - maximum number of relays to listen. */ constructor ({ libp2p, maxListeners = 1 }) { this._libp2p = libp2p @@ -50,10 +51,11 @@ class AutoRelay { * If the protocol is not supported, check if it was supported before and remove it as a listen relay. * If the protocol is supported, check if the peer supports **HOP** and add it as a listener if * inside the threshold. + * * @param {Object} props * @param {PeerId} props.peerId * @param {Array} props.protocols - * @return {Promise} + * @returns {Promise} */ async _onProtocolChange ({ peerId, protocols }) { const id = peerId.toB58String() @@ -92,8 +94,9 @@ class AutoRelay { /** * Peer disconnects. - * @param {Connection} connection connection to the peer - * @return {void} + * + * @param {Connection} connection - connection to the peer + * @returns {void} */ _onPeerDisconnected (connection) { const peerId = connection.remotePeer @@ -109,10 +112,11 @@ class AutoRelay { /** * Attempt to listen on the given relay connection. + * * @private - * @param {Connection} connection connection to the peer - * @param {string} id peer identifier string - * @return {Promise} + * @param {Connection} connection - connection to the peer + * @param {string} id - peer identifier string + * @returns {Promise} */ async _addListenRelay (connection, id) { // Check if already listening on enough relays @@ -152,9 +156,10 @@ class AutoRelay { /** * Remove listen relay. + * * @private - * @param {string} id peer identifier string. - * @return {void} + * @param {string} id - peer identifier string. + * @returns {void} */ _removeListenRelay (id) { if (this._listenRelays.delete(id)) { @@ -169,8 +174,9 @@ class AutoRelay { * 1. Check the metadata store for known relays, try to listen on the ones we are already connected. * 2. Dial and try to listen on the peers we know that support hop but are not connected. * 3. Search the network. + * * @param {Array} [peersToIgnore] - * @return {Promise} + * @returns {Promise} */ async _listenOnAvailableHopRelays (peersToIgnore = []) { // TODO: The peer redial issue on disconnect should be handled by connection gating diff --git a/src/circuit/circuit/hop.js b/src/circuit/circuit/hop.js index 114e2768fe..c653a7c9ae 100644 --- a/src/circuit/circuit/hop.js +++ b/src/circuit/circuit/hop.js @@ -118,8 +118,9 @@ module.exports.hop = async function hop ({ /** * Performs a CAN_HOP request to a relay peer, in order to understand its capabilities. + * * @param {object} options - * @param {Connection} options.connection Connection to the relay + * @param {Connection} options.connection - Connection to the relay * @returns {Promise} */ module.exports.canHop = async function canHop ({ diff --git a/src/identify/index.js b/src/identify/index.js index d7202ed9c4..53c2cee8f2 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -128,6 +128,7 @@ class IdentifyService { /** * Calls `push` for all peers in the `peerStore` that are connected + * * @returns {void} */ pushToPeerStore () { diff --git a/src/record/utils.js b/src/record/utils.js index 509fea7eec..65696156b8 100644 --- a/src/record/utils.js +++ b/src/record/utils.js @@ -5,6 +5,7 @@ const PeerRecord = require('./peer-record') /** * Create (or update if existing) self peer record and store it in the AddressBook. + * * @param {libp2p} libp2p * @returns {Promise} */ diff --git a/src/transport-manager.js b/src/transport-manager.js index 039d069fd8..7a47a9e90a 100644 --- a/src/transport-manager.js +++ b/src/transport-manager.js @@ -141,7 +141,7 @@ class TransportManager { * Starts listeners for each listen Multiaddr. * * @async - * @param {Array} addrs addresses to attempt to listen on + * @param {Array} addrs - addresses to attempt to listen on */ async listen (addrs) { if (!addrs || addrs.length === 0) { From e6b0134299d9ba3fd3c8b4ef0ebb860eda9d7034 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 11 Sep 2020 19:02:56 +0200 Subject: [PATCH 09/17] feat: auto relay network query for new relays --- package.json | 3 +- src/circuit/auto-relay.js | 58 ++++++++-- src/circuit/constants.js | 12 ++ src/circuit/index.js | 211 ++++++++-------------------------- src/circuit/transport.js | 194 +++++++++++++++++++++++++++++++ src/circuit/utils.js | 16 +++ src/config.js | 5 + src/index.js | 11 +- test/relay/auto-relay.node.js | 132 ++++++++++++++++++++- 9 files changed, 458 insertions(+), 184 deletions(-) create mode 100644 src/circuit/constants.js create mode 100644 src/circuit/transport.js create mode 100644 src/circuit/utils.js diff --git a/package.json b/package.json index 41587182fa..42c6c2ae87 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "aggregate-error": "^3.0.1", "any-signal": "^1.1.0", "bignumber.js": "^9.0.0", + "cids": "^1.0.0", "class-is": "^1.1.0", "debug": "^4.1.1", "err-code": "^2.0.0", @@ -66,6 +67,7 @@ "moving-average": "^1.0.0", "multiaddr": "^8.1.0", "multicodec": "^2.0.0", + "multihashing-async": "^2.0.1", "multistream-select": "^1.0.0", "mutable-proxy": "^1.0.0", "node-forge": "^0.9.1", @@ -89,7 +91,6 @@ "chai-as-promised": "^7.1.1", "chai-bytes": "^0.1.2", "chai-string": "^1.5.0", - "cids": "^1.0.0", "delay": "^4.3.0", "dirty-chai": "^2.0.1", "interop-libp2p": "^0.3.0", diff --git a/src/circuit/auto-relay.js b/src/circuit/auto-relay.js index c0dafc4b17..c9559ce9a1 100644 --- a/src/circuit/auto-relay.js +++ b/src/circuit/auto-relay.js @@ -11,10 +11,13 @@ const PeerId = require('peer-id') const { relay: multicodec } = require('./multicodec') const { canHop } = require('./circuit/hop') - -const circuitProtoCode = 290 -const hopMetadataKey = 'hop_relay' -const hopMetadataValue = 'true' +const { namespaceToCid } = require('./utils') +const { + CIRCUIT_PROTO_CODE, + HOP_METADATA_KEY, + HOP_METADATA_VALUE, + RELAY_RENDEZVOUS_NS +} = require('./constants') class AutoRelay { /** @@ -76,7 +79,7 @@ class AutoRelay { const connection = this._connectionManager.get(peerId) // Do not hop on a relayed connection - if (connection.remoteAddr.protoCodes().includes(circuitProtoCode)) { + if (connection.remoteAddr.protoCodes().includes(CIRCUIT_PROTO_CODE)) { log(`relayed connection to ${id} will not be used to hop on`) return } @@ -84,7 +87,7 @@ class AutoRelay { const supportsHop = await canHop({ connection }) if (supportsHop) { - this._peerStore.metadataBook.set(peerId, hopMetadataKey, uint8ArrayFromString(hopMetadataValue)) + this._peerStore.metadataBook.set(peerId, HOP_METADATA_KEY, uint8ArrayFromString(HOP_METADATA_VALUE)) await this._addListenRelay(connection, id) } } catch (err) { @@ -125,15 +128,21 @@ class AutoRelay { } // Create relay listen addr - let listenAddr, remoteMultiaddr + let listenAddr, remoteMultiaddr, remoteAddrs try { - const remoteAddrs = this._peerStore.addressBook.get(connection.remotePeer) + remoteAddrs = this._peerStore.addressBook.get(connection.remotePeer) // TODO: HOP Relays should avoid advertising private addresses! remoteMultiaddr = remoteAddrs.find(a => a.isCertified).multiaddr // Get first announced address certified } catch (_) { log.error(`${id} does not have announced certified multiaddrs`) - return + + // Attempt first if existing + if (!remoteAddrs || !remoteAddrs.length) { + return + } + + remoteMultiaddr = remoteAddrs[0].multiaddr } if (!remoteMultiaddr.protoNames().includes('p2p')) { @@ -194,10 +203,10 @@ class AutoRelay { continue } - const supportsHop = metadataMap.get(hopMetadataKey) + const supportsHop = metadataMap.get(HOP_METADATA_KEY) // Continue to next if it does not support Hop - if (!supportsHop || uint8ArrayToString(supportsHop) !== hopMetadataValue) { + if (!supportsHop || uint8ArrayToString(supportsHop) !== HOP_METADATA_VALUE) { continue } @@ -229,7 +238,32 @@ class AutoRelay { } } - // TODO: Try to find relays to hop on the network + // Try to find relays to hop on the network + try { + const cid = await namespaceToCid(RELAY_RENDEZVOUS_NS) + for await (const provider of this._libp2p.contentRouting.findProviders(cid)) { + if (!provider || !provider.id || !provider.multiaddrs || !provider.multiaddrs.length) { + continue + } + const peerId = provider.id + + this._peerStore.addressBook.add(peerId, provider.multiaddrs) + const connection = await this._libp2p.dial(peerId) + + await this._addListenRelay(connection, peerId.toB58String()) + + // Check if already listening on enough relays + if (this._listenRelays.size >= this.maxListeners) { + return + } + } + } catch (err) { + if (err.code !== 'NO_ROUTERS_AVAILABLE') { + throw err + } else { + log('there are no routers configured to find hop relay services') + } + } } } diff --git a/src/circuit/constants.js b/src/circuit/constants.js new file mode 100644 index 0000000000..53bd6505d6 --- /dev/null +++ b/src/circuit/constants.js @@ -0,0 +1,12 @@ +'use strict' + +const minute = 60 * 1000 + +module.exports = { + ADVERTISE_BOOT_DELAY: 15 * minute, + ADVERTISE_TTL: 30 * minute, + CIRCUIT_PROTO_CODE: 290, + HOP_METADATA_KEY: 'hop_relay', + HOP_METADATA_VALUE: 'true', + RELAY_RENDEZVOUS_NS: '/libp2p/relay' +} diff --git a/src/circuit/index.js b/src/circuit/index.js index 705dcdad82..d0df9041ae 100644 --- a/src/circuit/index.js +++ b/src/circuit/index.js @@ -1,197 +1,76 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:circuit') -log.error = debug('libp2p:circuit:error') - -const mafmt = require('mafmt') -const multiaddr = require('multiaddr') -const PeerId = require('peer-id') -const withIs = require('class-is') -const { CircuitRelay: CircuitPB } = require('./protocol') - -const toConnection = require('libp2p-utils/src/stream-to-ma-conn') +const log = debug('libp2p:relay') +log.error = debug('libp2p:relay:error') const AutoRelay = require('./auto-relay') -const { relay: multicodec } = require('./multicodec') -const createListener = require('./listener') -const { handleCanHop, handleHop, hop } = require('./circuit/hop') -const { handleStop } = require('./circuit/stop') -const StreamHandler = require('./circuit/stream-handler') - -class Circuit { +const { namespaceToCid } = require('./utils') +const { + ADVERTISE_BOOT_DELAY, + ADVERTISE_TTL, + RELAY_RENDEZVOUS_NS +} = require('./constants') + +class Relay { /** - * Creates an instance of Circuit. + * Creates an instance of Relay. * * @class - * @param {object} options - * @param {Libp2p} options.libp2p - * @param {Upgrader} options.upgrader + * @param {Libp2p} libp2p */ - constructor ({ libp2p, upgrader }) { - this._dialer = libp2p.dialer - this._registrar = libp2p.registrar - this._connectionManager = libp2p.connectionManager - this._upgrader = upgrader + constructor (libp2p) { this._options = libp2p._config.relay this._libp2p = libp2p - this.peerId = libp2p.peerId - this._registrar.handle(multicodec, this._onProtocol.bind(this)) // Create autoRelay if enabled this._autoRelay = this._options.autoRelay.enabled && new AutoRelay({ libp2p, ...this._options.autoRelay }) } - async _onProtocol ({ connection, stream }) { - const streamHandler = new StreamHandler({ stream }) - const request = await streamHandler.read() - - if (!request) { - return - } - - const circuit = this - let virtualConnection - - switch (request.type) { - case CircuitPB.Type.CAN_HOP: { - log('received CAN_HOP request from %s', connection.remotePeer.toB58String()) - await handleCanHop({ circuit, connection, streamHandler }) - break - } - case CircuitPB.Type.HOP: { - log('received HOP request from %s', connection.remotePeer.toB58String()) - virtualConnection = await handleHop({ - connection, - request, - streamHandler, - circuit - }) - break - } - case CircuitPB.Type.STOP: { - log('received STOP request from %s', connection.remotePeer.toB58String()) - virtualConnection = await handleStop({ - connection, - request, - streamHandler, - circuit - }) - break - } - default: { - log('Request of type %s not supported', request.type) - } - } - - if (virtualConnection) { - const remoteAddr = multiaddr(request.dstPeer.addrs[0]) - const localAddr = multiaddr(request.srcPeer.addrs[0]) - const maConn = toConnection({ - stream: virtualConnection, - remoteAddr, - localAddr - }) - const type = CircuitPB.Type === CircuitPB.Type.HOP ? 'relay' : 'inbound' - log('new %s connection %s', type, maConn.remoteAddr) - - const conn = await this._upgrader.upgradeInbound(maConn) - log('%s connection %s upgraded', type, maConn.remoteAddr) - this.handler && this.handler(conn) - } - } - /** - * Dial a peer over a relay - * - * @param {multiaddr} ma - the multiaddr of the peer to dial - * @param {Object} options - dial options - * @param {AbortSignal} [options.signal] - An optional abort signal - * @returns {Connection} - the connection + * Start Relay service. + * @returns {void} */ - async dial (ma, options) { - // Check the multiaddr to see if it contains a relay and a destination peer - const addrs = ma.toString().split('/p2p-circuit') - const relayAddr = multiaddr(addrs[0]) - const destinationAddr = multiaddr(addrs[addrs.length - 1]) - const relayPeer = PeerId.createFromCID(relayAddr.getPeerId()) - const destinationPeer = PeerId.createFromCID(destinationAddr.getPeerId()) - - let disconnectOnFailure = false - let relayConnection = this._connectionManager.get(relayPeer) - if (!relayConnection) { - relayConnection = await this._dialer.connectToPeer(relayAddr, options) - disconnectOnFailure = true - } - - try { - const virtualConnection = await hop({ - connection: relayConnection, - circuit: this, - request: { - type: CircuitPB.Type.HOP, - srcPeer: { - id: this.peerId.toBytes(), - addrs: this._libp2p.multiaddrs.map(addr => addr.bytes) - }, - dstPeer: { - id: destinationPeer.toBytes(), - addrs: [multiaddr(destinationAddr).bytes] - } - } - }) - - const localAddr = relayAddr.encapsulate(`/p2p-circuit/p2p/${this.peerId.toB58String()}`) - const maConn = toConnection({ - stream: virtualConnection, - remoteAddr: ma, - localAddr - }) - log('new outbound connection %s', maConn.remoteAddr) - - return this._upgrader.upgradeOutbound(maConn) - } catch (err) { - log.error('Circuit relay dial failed', err) - disconnectOnFailure && await relayConnection.close() - throw err + start () { + // Advertise service if HOP enabled + const canHop = this._options.hop.enabled + + if (canHop) { + this._timeout = setTimeout(() => { + this._advertiseService() + }, this._options.advertise.bootDelay || ADVERTISE_BOOT_DELAY) } } /** - * Create a listener - * - * @param {any} options - * @param {Function} handler - * @returns {listener} + * Stop Relay service. + * @returns {void} */ - createListener (options, handler) { - if (typeof options === 'function') { - handler = options - options = {} - } - - // Called on successful HOP and STOP requests - this.handler = handler - - return createListener(this._libp2p, options) + stop () { + clearTimeout(this._timeout) } /** - * Filter check for all Multiaddrs that this transport can dial on - * - * @param {Array} multiaddrs - * @returns {Array} + * Advertise hop relay service in the network. + * @returns {Promise} */ - filter (multiaddrs) { - multiaddrs = Array.isArray(multiaddrs) ? multiaddrs : [multiaddrs] + async _advertiseService () { + try { + const cid = await namespaceToCid(RELAY_RENDEZVOUS_NS) + await this._libp2p.contentRouting.provide(cid) + } catch (err) { + if (err.code === 'NO_ROUTERS_AVAILABLE') { + log('there are no routers configured to advertise hop relay service') + } else { + log.error(err) + } + } - return multiaddrs.filter((ma) => { - return mafmt.Circuit.matches(ma) - }) + // Restart timeout + this._timeout = setTimeout(() => { + this._advertiseService() + }, this._options.advertise.ttl || ADVERTISE_TTL) } } -/** - * @type {Circuit} - */ -module.exports = withIs(Circuit, { className: 'Circuit', symbolName: '@libp2p/js-libp2p-circuit/circuit' }) +module.exports = Relay diff --git a/src/circuit/transport.js b/src/circuit/transport.js new file mode 100644 index 0000000000..e876fa6005 --- /dev/null +++ b/src/circuit/transport.js @@ -0,0 +1,194 @@ +'use strict' + +const debug = require('debug') +const log = debug('libp2p:circuit') +log.error = debug('libp2p:circuit:error') + +const mafmt = require('mafmt') +const multiaddr = require('multiaddr') +const PeerId = require('peer-id') +const withIs = require('class-is') +const { CircuitRelay: CircuitPB } = require('./protocol') + +const toConnection = require('libp2p-utils/src/stream-to-ma-conn') + +const { relay: multicodec } = require('./multicodec') +const createListener = require('./listener') +const { handleCanHop, handleHop, hop } = require('./circuit/hop') +const { handleStop } = require('./circuit/stop') +const StreamHandler = require('./circuit/stream-handler') + +class Circuit { + /** + * Creates an instance of the Circuit Transport. + * + * @constructor + * @param {object} options + * @param {Libp2p} options.libp2p + * @param {Upgrader} options.upgrader + */ + constructor ({ libp2p, upgrader }) { + this._dialer = libp2p.dialer + this._registrar = libp2p.registrar + this._connectionManager = libp2p.connectionManager + this._upgrader = upgrader + this._options = libp2p._config.relay + this._libp2p = libp2p + this.peerId = libp2p.peerId + + this._registrar.handle(multicodec, this._onProtocol.bind(this)) + } + + async _onProtocol ({ connection, stream }) { + const streamHandler = new StreamHandler({ stream }) + const request = await streamHandler.read() + + if (!request) { + return + } + + const circuit = this + let virtualConnection + + switch (request.type) { + case CircuitPB.Type.CAN_HOP: { + log('received CAN_HOP request from %s', connection.remotePeer.toB58String()) + await handleCanHop({ circuit, connection, streamHandler }) + break + } + case CircuitPB.Type.HOP: { + log('received HOP request from %s', connection.remotePeer.toB58String()) + virtualConnection = await handleHop({ + connection, + request, + streamHandler, + circuit + }) + break + } + case CircuitPB.Type.STOP: { + log('received STOP request from %s', connection.remotePeer.toB58String()) + virtualConnection = await handleStop({ + connection, + request, + streamHandler, + circuit + }) + break + } + default: { + log('Request of type %s not supported', request.type) + } + } + + if (virtualConnection) { + const remoteAddr = multiaddr(request.dstPeer.addrs[0]) + const localAddr = multiaddr(request.srcPeer.addrs[0]) + const maConn = toConnection({ + stream: virtualConnection, + remoteAddr, + localAddr + }) + const type = CircuitPB.Type === CircuitPB.Type.HOP ? 'relay' : 'inbound' + log('new %s connection %s', type, maConn.remoteAddr) + + const conn = await this._upgrader.upgradeInbound(maConn) + log('%s connection %s upgraded', type, maConn.remoteAddr) + this.handler && this.handler(conn) + } + } + + /** + * Dial a peer over a relay + * + * @param {multiaddr} ma - the multiaddr of the peer to dial + * @param {Object} options - dial options + * @param {AbortSignal} [options.signal] - An optional abort signal + * @returns {Connection} - the connection + */ + async dial (ma, options) { + // Check the multiaddr to see if it contains a relay and a destination peer + const addrs = ma.toString().split('/p2p-circuit') + const relayAddr = multiaddr(addrs[0]) + const destinationAddr = multiaddr(addrs[addrs.length - 1]) + const relayPeer = PeerId.createFromCID(relayAddr.getPeerId()) + const destinationPeer = PeerId.createFromCID(destinationAddr.getPeerId()) + + let disconnectOnFailure = false + let relayConnection = this._connectionManager.get(relayPeer) + if (!relayConnection) { + relayConnection = await this._dialer.connectToPeer(relayAddr, options) + disconnectOnFailure = true + } + + try { + const virtualConnection = await hop({ + connection: relayConnection, + circuit: this, + request: { + type: CircuitPB.Type.HOP, + srcPeer: { + id: this.peerId.toBytes(), + addrs: this._libp2p.multiaddrs.map(addr => addr.bytes) + }, + dstPeer: { + id: destinationPeer.toBytes(), + addrs: [multiaddr(destinationAddr).bytes] + } + } + }) + + const localAddr = relayAddr.encapsulate(`/p2p-circuit/p2p/${this.peerId.toB58String()}`) + const maConn = toConnection({ + stream: virtualConnection, + remoteAddr: ma, + localAddr + }) + log('new outbound connection %s', maConn.remoteAddr) + + return this._upgrader.upgradeOutbound(maConn) + } catch (err) { + log.error('Circuit relay dial failed', err) + disconnectOnFailure && await relayConnection.close() + throw err + } + } + + /** + * Create a listener + * + * @param {any} options + * @param {Function} handler + * @return {listener} + */ + createListener (options, handler) { + if (typeof options === 'function') { + handler = options + options = {} + } + + // Called on successful HOP and STOP requests + this.handler = handler + + return createListener(this._libp2p, options) + } + + /** + * Filter check for all Multiaddrs that this transport can dial on + * + * @param {Array} multiaddrs + * @returns {Array} + */ + filter (multiaddrs) { + multiaddrs = Array.isArray(multiaddrs) ? multiaddrs : [multiaddrs] + + return multiaddrs.filter((ma) => { + return mafmt.Circuit.matches(ma) + }) + } +} + +/** + * @type {Circuit} + */ +module.exports = withIs(Circuit, { className: 'Circuit', symbolName: '@libp2p/js-libp2p-circuit/circuit' }) diff --git a/src/circuit/utils.js b/src/circuit/utils.js new file mode 100644 index 0000000000..7426271cb7 --- /dev/null +++ b/src/circuit/utils.js @@ -0,0 +1,16 @@ +'use strict' + +const CID = require('cids') +const multihashing = require('multihashing-async') + +/** + * Convert a namespace string into a cid. + * @param {string} namespace + * @return {Promise} + */ +module.exports.namespaceToCid = async (namespace) => { + const bytes = new TextEncoder('utf8').encode(namespace) + const hash = await multihashing(bytes, 'sha2-256') + + return new CID(hash) +} diff --git a/src/config.js b/src/config.js index eb28e3c734..512ce72717 100644 --- a/src/config.js +++ b/src/config.js @@ -4,6 +4,7 @@ const mergeOptions = require('merge-options') const { dnsaddrResolver } = require('multiaddr/src/resolvers') const Constants = require('./constants') +const RelayConstants = require('./circuit/constants') const { FaultTolerance } = require('./transport-manager') @@ -56,6 +57,10 @@ const DefaultConfig = { }, relay: { enabled: true, + advertise: { + bootDelay: RelayConstants.ADVERTISE_BOOT_DELAY, + ttl: RelayConstants.ADVERTISE_TTL + }, hop: { enabled: false, active: false diff --git a/src/index.js b/src/index.js index 2e46ebd5d7..f3c4b326c3 100644 --- a/src/index.js +++ b/src/index.js @@ -17,7 +17,8 @@ const { codes, messages } = require('./errors') const AddressManager = require('./address-manager') const ConnectionManager = require('./connection-manager') -const Circuit = require('./circuit') +const Circuit = require('./circuit/transport') +const Relay = require('./circuit') const Dialer = require('./dialer') const Keychain = require('./keychain') const Metrics = require('./metrics') @@ -146,6 +147,7 @@ class Libp2p extends EventEmitter { if (this._config.relay.enabled) { this.transportManager.add(Circuit.prototype[Symbol.toStringTag], Circuit) + this.relay = new Relay(this) } // Attach stream multiplexers @@ -249,6 +251,10 @@ class Libp2p extends EventEmitter { try { this._isStarted = false + + // Relay + this.relay && this.relay.stop() + for (const service of this._discovery.values()) { service.removeListener('peer', this._onDiscoveryPeer) } @@ -503,6 +509,9 @@ class Libp2p extends EventEmitter { // Peer discovery await this._setupPeerDiscovery() + + // Relay + this.relay && this.relay.start() } /** diff --git a/test/relay/auto-relay.node.js b/test/relay/auto-relay.node.js index 96f94cd7bb..cd0add3771 100644 --- a/test/relay/auto-relay.node.js +++ b/test/relay/auto-relay.node.js @@ -8,7 +8,10 @@ const { expect } = chai const delay = require('delay') const pWaitFor = require('p-wait-for') const sinon = require('sinon') +const nock = require('nock') +const ipfsHttpClient = require('ipfs-http-client') +const DelegatedContentRouter = require('libp2p-delegated-content-routing') const multiaddr = require('multiaddr') const Libp2p = require('../../src') const { relay: relayMulticodec } = require('../../src/circuit/multicodec') @@ -59,7 +62,7 @@ describe('auto-relay', () => { }) }) - autoRelay = libp2p.transportManager._transports.get('Circuit')._autoRelay + autoRelay = libp2p.relay._autoRelay expect(autoRelay.maxListeners).to.eql(1) }) @@ -144,7 +147,7 @@ describe('auto-relay', () => { }) }) - autoRelay1 = relayLibp2p1.transportManager._transports.get('Circuit')._autoRelay + autoRelay1 = relayLibp2p1.relay._autoRelay expect(autoRelay1.maxListeners).to.eql(1) }) @@ -412,8 +415,8 @@ describe('auto-relay', () => { }) }) - autoRelay1 = relayLibp2p1.transportManager._transports.get('Circuit')._autoRelay - autoRelay2 = relayLibp2p2.transportManager._transports.get('Circuit')._autoRelay + autoRelay1 = relayLibp2p1.relay._autoRelay + autoRelay2 = relayLibp2p2.relay._autoRelay }) beforeEach(() => { @@ -457,4 +460,125 @@ describe('auto-relay', () => { expect(autoRelay1._listenRelays.size).to.equal(1) }) }) + + describe('discovery', () => { + let libp2p + let libp2p2 + let relayLibp2p + + beforeEach(async () => { + const peerIds = await createPeerId({ number: 3 }) + + // Create 2 nodes, and turn HOP on for the relay + ;[libp2p, libp2p2, relayLibp2p] = peerIds.map((peerId, index) => { + const delegate = new DelegatedContentRouter(peerId, ipfsHttpClient({ + host: '0.0.0.0', + protocol: 'http', + port: 60197 + }), [ + multiaddr('/ip4/0.0.0.0/tcp/60197') + ]) + + const opts = { + ...baseOptions, + config: { + ...baseOptions.config, + relay: { + advertise: { + bootDelay: 1000, + ttl: 1000 + }, + hop: { + enabled: index === 2 + }, + autoRelay: { + enabled: true, + maxListeners: 1 + } + } + } + } + + return new Libp2p({ + ...opts, + modules: { + ...opts.modules, + contentRouting: [delegate] + }, + addresses: { + listen: [listenAddr] + }, + connectionManager: { + autoDial: false + }, + peerDiscovery: { + autoDial: false + }, + peerId + }) + }) + + sinon.spy(relayLibp2p.contentRouting, 'provide') + }) + + beforeEach(async () => { + nock('http://0.0.0.0:60197') + // mock the refs call + .post('/api/v0/refs') + .query(true) + .reply(200, null, [ + 'Content-Type', 'application/json', + 'X-Chunked-Output', '1' + ]) + + // Start each node + await Promise.all([libp2p, libp2p2, relayLibp2p].map(libp2p => libp2p.start())) + + // Should provide on start + await pWaitFor(() => relayLibp2p.contentRouting.provide.callCount === 1) + + const provider = relayLibp2p.peerId.toB58String() + const multiaddrs = relayLibp2p.multiaddrs.map((m) => m.toString()) + + // Mock findProviders + nock('http://0.0.0.0:60197') + .post('/api/v0/dht/findprovs') + .query(true) + .reply(200, `{"Extra":"","ID":"${provider}","Responses":[{"Addrs":${JSON.stringify(multiaddrs)},"ID":"${provider}"}],"Type":4}\n`, [ + 'Content-Type', 'application/json', + 'X-Chunked-Output', '1' + ]) + }) + + afterEach(() => { + // Stop each node + return Promise.all([libp2p, libp2p2, relayLibp2p].map(libp2p => libp2p.stop())) + }) + + it('should find providers for relay and add it as listen relay', async () => { + const originalMultiaddrsLength = libp2p.multiaddrs.length + + // Spy add listen relay + sinon.spy(libp2p.relay._autoRelay, '_addListenRelay') + // Spy Find Providers + sinon.spy(libp2p.contentRouting, 'findProviders') + + // Try to listen on Available hop relays + await libp2p.relay._autoRelay._listenOnAvailableHopRelays() + + // Should try to find relay service providers + await pWaitFor(() => libp2p.contentRouting.findProviders.callCount === 1) + // Wait for peer added as listen relay + await pWaitFor(() => libp2p.relay._autoRelay._addListenRelay.callCount === 1) + expect(libp2p.relay._autoRelay._listenRelays.size).to.equal(1) + await pWaitFor(() => libp2p.multiaddrs.length === originalMultiaddrsLength + 1) + + const relayedAddr = libp2p.multiaddrs[libp2p.multiaddrs.length - 1] + libp2p2.peerStore.addressBook.set(libp2p2.peerId, [relayedAddr]) + + // Dial from peer 2 through the relayed address + const conn = await libp2p2.dial(libp2p2.peerId) + expect(conn).to.exist() + }) + }) }) From 3d2181f6fa350bb895b3c511b1da0b8501718f2b Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 23 Sep 2020 16:40:33 +0200 Subject: [PATCH 10/17] chore: address review --- src/circuit/auto-relay.js | 2 +- src/circuit/constants.js | 12 ++++++------ src/circuit/index.js | 20 ++++++++++++++++---- src/config.js | 1 + test/relay/auto-relay.node.js | 34 +++++++++++++++++----------------- 5 files changed, 41 insertions(+), 28 deletions(-) diff --git a/src/circuit/auto-relay.js b/src/circuit/auto-relay.js index c9559ce9a1..4bffecc06d 100644 --- a/src/circuit/auto-relay.js +++ b/src/circuit/auto-relay.js @@ -242,7 +242,7 @@ class AutoRelay { try { const cid = await namespaceToCid(RELAY_RENDEZVOUS_NS) for await (const provider of this._libp2p.contentRouting.findProviders(cid)) { - if (!provider || !provider.id || !provider.multiaddrs || !provider.multiaddrs.length) { + if (!provider || !provider.multiaddrs.length) { continue } const peerId = provider.id diff --git a/src/circuit/constants.js b/src/circuit/constants.js index 53bd6505d6..b4de629c63 100644 --- a/src/circuit/constants.js +++ b/src/circuit/constants.js @@ -3,10 +3,10 @@ const minute = 60 * 1000 module.exports = { - ADVERTISE_BOOT_DELAY: 15 * minute, - ADVERTISE_TTL: 30 * minute, - CIRCUIT_PROTO_CODE: 290, - HOP_METADATA_KEY: 'hop_relay', - HOP_METADATA_VALUE: 'true', - RELAY_RENDEZVOUS_NS: '/libp2p/relay' + ADVERTISE_BOOT_DELAY: 15 * minute, // Delay before HOP relay service is advertised on the network + ADVERTISE_TTL: 30 * minute, // Delay Between HOP relay service advertisements on the network + CIRCUIT_PROTO_CODE: 290, // Multicodec code + HOP_METADATA_KEY: 'hop_relay', // PeerStore metadaBook key for HOP relay service + HOP_METADATA_VALUE: 'true', // PeerStore metadaBook value for HOP relay service + RELAY_RENDEZVOUS_NS: '/libp2p/relay' // Relay HOP relay service namespace for discovery } diff --git a/src/circuit/index.js b/src/circuit/index.js index d0df9041ae..ae54ce1863 100644 --- a/src/circuit/index.js +++ b/src/circuit/index.js @@ -20,8 +20,16 @@ class Relay { * @param {Libp2p} libp2p */ constructor (libp2p) { - this._options = libp2p._config.relay this._libp2p = libp2p + this._options = { + advertise: { + bootDelay: ADVERTISE_BOOT_DELAY, + enabled: true, + ttl: ADVERTISE_TTL, + ...libp2p._config.relay.advertise + }, + ...libp2p._config.relay + } // Create autoRelay if enabled this._autoRelay = this._options.autoRelay.enabled && new AutoRelay({ libp2p, ...this._options.autoRelay }) @@ -35,10 +43,10 @@ class Relay { // Advertise service if HOP enabled const canHop = this._options.hop.enabled - if (canHop) { + if (canHop && this._options.advertise.enabled) { this._timeout = setTimeout(() => { this._advertiseService() - }, this._options.advertise.bootDelay || ADVERTISE_BOOT_DELAY) + }, this._options.advertise.bootDelay) } } @@ -64,12 +72,16 @@ class Relay { } else { log.error(err) } + // Stop the advertise + this.stop() + + return } // Restart timeout this._timeout = setTimeout(() => { this._advertiseService() - }, this._options.advertise.ttl || ADVERTISE_TTL) + }, this._options.advertise.ttl) } } diff --git a/src/config.js b/src/config.js index 512ce72717..cc45dee5aa 100644 --- a/src/config.js +++ b/src/config.js @@ -59,6 +59,7 @@ const DefaultConfig = { enabled: true, advertise: { bootDelay: RelayConstants.ADVERTISE_BOOT_DELAY, + enabled: true, ttl: RelayConstants.ADVERTISE_TTL }, hop: { diff --git a/test/relay/auto-relay.node.js b/test/relay/auto-relay.node.js index cd0add3771..43f42a6ab8 100644 --- a/test/relay/auto-relay.node.js +++ b/test/relay/auto-relay.node.js @@ -462,15 +462,15 @@ describe('auto-relay', () => { }) describe('discovery', () => { - let libp2p - let libp2p2 + let local + let remote let relayLibp2p beforeEach(async () => { const peerIds = await createPeerId({ number: 3 }) // Create 2 nodes, and turn HOP on for the relay - ;[libp2p, libp2p2, relayLibp2p] = peerIds.map((peerId, index) => { + ;[local, remote, relayLibp2p] = peerIds.map((peerId, index) => { const delegate = new DelegatedContentRouter(peerId, ipfsHttpClient({ host: '0.0.0.0', protocol: 'http', @@ -532,7 +532,7 @@ describe('auto-relay', () => { ]) // Start each node - await Promise.all([libp2p, libp2p2, relayLibp2p].map(libp2p => libp2p.start())) + await Promise.all([local, remote, relayLibp2p].map(libp2p => libp2p.start())) // Should provide on start await pWaitFor(() => relayLibp2p.contentRouting.provide.callCount === 1) @@ -552,32 +552,32 @@ describe('auto-relay', () => { afterEach(() => { // Stop each node - return Promise.all([libp2p, libp2p2, relayLibp2p].map(libp2p => libp2p.stop())) + return Promise.all([local, remote, relayLibp2p].map(libp2p => libp2p.stop())) }) it('should find providers for relay and add it as listen relay', async () => { - const originalMultiaddrsLength = libp2p.multiaddrs.length + const originalMultiaddrsLength = local.multiaddrs.length // Spy add listen relay - sinon.spy(libp2p.relay._autoRelay, '_addListenRelay') + sinon.spy(local.relay._autoRelay, '_addListenRelay') // Spy Find Providers - sinon.spy(libp2p.contentRouting, 'findProviders') + sinon.spy(local.contentRouting, 'findProviders') // Try to listen on Available hop relays - await libp2p.relay._autoRelay._listenOnAvailableHopRelays() + await local.relay._autoRelay._listenOnAvailableHopRelays() // Should try to find relay service providers - await pWaitFor(() => libp2p.contentRouting.findProviders.callCount === 1) + await pWaitFor(() => local.contentRouting.findProviders.callCount === 1) // Wait for peer added as listen relay - await pWaitFor(() => libp2p.relay._autoRelay._addListenRelay.callCount === 1) - expect(libp2p.relay._autoRelay._listenRelays.size).to.equal(1) - await pWaitFor(() => libp2p.multiaddrs.length === originalMultiaddrsLength + 1) + await pWaitFor(() => local.relay._autoRelay._addListenRelay.callCount === 1) + expect(local.relay._autoRelay._listenRelays.size).to.equal(1) + await pWaitFor(() => local.multiaddrs.length === originalMultiaddrsLength + 1) - const relayedAddr = libp2p.multiaddrs[libp2p.multiaddrs.length - 1] - libp2p2.peerStore.addressBook.set(libp2p2.peerId, [relayedAddr]) + const relayedAddr = local.multiaddrs[local.multiaddrs.length - 1] + remote.peerStore.addressBook.set(local.peerId, [relayedAddr]) - // Dial from peer 2 through the relayed address - const conn = await libp2p2.dial(libp2p2.peerId) + // Dial from remote through the relayed address + const conn = await remote.dial(local.peerId) expect(conn).to.exist() }) }) From 29e30c2199c72f0c8a31de57a436632c9f2fabe0 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 23 Sep 2020 16:47:03 +0200 Subject: [PATCH 11/17] chore: add configuration docs for auto relay and hop service --- doc/CONFIGURATION.md | 32 ++++++++++++++++++++++++++++++++ src/circuit/auto-relay.js | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/doc/CONFIGURATION.md b/doc/CONFIGURATION.md index 527ee35733..1e03715384 100644 --- a/doc/CONFIGURATION.md +++ b/doc/CONFIGURATION.md @@ -20,6 +20,7 @@ - [Customizing DHT](#customizing-dht) - [Setup with Content and Peer Routing](#setup-with-content-and-peer-routing) - [Setup with Relay](#setup-with-relay) + - [Setup with Auto Relay](#setup-with-auto-relay) - [Setup with Keychain](#setup-with-keychain) - [Configuring Dialing](#configuring-dialing) - [Configuring Connection Manager](#configuring-connection-manager) @@ -419,12 +420,43 @@ const node = await Libp2p.create({ hop: { enabled: true, // Allows you to be a relay for other peers active: true // You will attempt to dial destination peers if you are not connected to them + }, + advertise: { + bootDelay: 15 * 60 * 1000, // Delay before HOP relay service is advertised on the network + enabled: true, // Allows you to disable the advertise of the Hop service + ttl: 30 * 60 * 1000 // Delay Between HOP relay service advertisements on the network } } } }) ``` +#### Setup with Auto Relay + +```js +const Libp2p = require('libp2p') +const TCP = require('libp2p-tcp') +const MPLEX = require('libp2p-mplex') +const SECIO = require('libp2p-secio') + +const node = await Libp2p.create({ + modules: { + transport: [TCP], + streamMuxer: [MPLEX], + connEncryption: [SECIO] + }, + config: { + relay: { // Circuit Relay options (this config is part of libp2p core configurations) + enabled: true, // Allows you to dial and accept relayed connections. Does not make you a relay. + autoRelay: { + enabled: false, // Allows you to bind to relays with HOP enabled for improving node dialability + maxListeners: 2 // Configure maximum number of HOP relays to use + }, + } + } +}) +``` + #### Setup with Keychain Libp2p allows you to setup a secure keychain to manage your keys. The keychain configuration object should have the following properties: diff --git a/src/circuit/auto-relay.js b/src/circuit/auto-relay.js index 4bffecc06d..c0475dbada 100644 --- a/src/circuit/auto-relay.js +++ b/src/circuit/auto-relay.js @@ -242,7 +242,7 @@ class AutoRelay { try { const cid = await namespaceToCid(RELAY_RENDEZVOUS_NS) for await (const provider of this._libp2p.contentRouting.findProviders(cid)) { - if (!provider || !provider.multiaddrs.length) { + if (!provider.multiaddrs.length) { continue } const peerId = provider.id From 2746b4b0259faf19d570e08de1eafa8225b4362b Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 5 Oct 2020 16:49:43 +0200 Subject: [PATCH 12/17] chore: apply suggestions from code review Co-authored-by: Jacob Heun --- doc/CONFIGURATION.md | 4 ++-- src/circuit/index.js | 6 +++--- src/config.js | 2 +- test/relay/auto-relay.node.js | 3 ++- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/CONFIGURATION.md b/doc/CONFIGURATION.md index 1e03715384..15f34d4ff6 100644 --- a/doc/CONFIGURATION.md +++ b/doc/CONFIGURATION.md @@ -449,9 +449,9 @@ const node = await Libp2p.create({ relay: { // Circuit Relay options (this config is part of libp2p core configurations) enabled: true, // Allows you to dial and accept relayed connections. Does not make you a relay. autoRelay: { - enabled: false, // Allows you to bind to relays with HOP enabled for improving node dialability + enabled: true, // Allows you to bind to relays with HOP enabled for improving node dialability maxListeners: 2 // Configure maximum number of HOP relays to use - }, + } } } }) diff --git a/src/circuit/index.js b/src/circuit/index.js index ae54ce1863..dbb70e61a4 100644 --- a/src/circuit/index.js +++ b/src/circuit/index.js @@ -68,12 +68,12 @@ class Relay { await this._libp2p.contentRouting.provide(cid) } catch (err) { if (err.code === 'NO_ROUTERS_AVAILABLE') { - log('there are no routers configured to advertise hop relay service') + log.error('a content router, such as a DHT, must be provided in order to advertise the relay service', err) + // Stop the advertise + this.stop() } else { log.error(err) } - // Stop the advertise - this.stop() return } diff --git a/src/config.js b/src/config.js index cc45dee5aa..2337e249ae 100644 --- a/src/config.js +++ b/src/config.js @@ -59,7 +59,7 @@ const DefaultConfig = { enabled: true, advertise: { bootDelay: RelayConstants.ADVERTISE_BOOT_DELAY, - enabled: true, + enabled: false, ttl: RelayConstants.ADVERTISE_TTL }, hop: { diff --git a/test/relay/auto-relay.node.js b/test/relay/auto-relay.node.js index 43f42a6ab8..8d0cdfd27e 100644 --- a/test/relay/auto-relay.node.js +++ b/test/relay/auto-relay.node.js @@ -486,7 +486,8 @@ describe('auto-relay', () => { relay: { advertise: { bootDelay: 1000, - ttl: 1000 + ttl: 1000, + enabled: true }, hop: { enabled: index === 2 From 722cacd6d22fb0c7f8b1e66dc60eb8dfc529eaf9 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 7 Oct 2020 17:58:11 +0200 Subject: [PATCH 13/17] chore: lint issues fixed --- src/circuit/index.js | 3 +++ src/circuit/transport.js | 4 ++-- src/circuit/utils.js | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/circuit/index.js b/src/circuit/index.js index dbb70e61a4..d12d882432 100644 --- a/src/circuit/index.js +++ b/src/circuit/index.js @@ -37,6 +37,7 @@ class Relay { /** * Start Relay service. + * * @returns {void} */ start () { @@ -52,6 +53,7 @@ class Relay { /** * Stop Relay service. + * * @returns {void} */ stop () { @@ -60,6 +62,7 @@ class Relay { /** * Advertise hop relay service in the network. + * * @returns {Promise} */ async _advertiseService () { diff --git a/src/circuit/transport.js b/src/circuit/transport.js index e876fa6005..cc79870564 100644 --- a/src/circuit/transport.js +++ b/src/circuit/transport.js @@ -22,7 +22,7 @@ class Circuit { /** * Creates an instance of the Circuit Transport. * - * @constructor + * @class * @param {object} options * @param {Libp2p} options.libp2p * @param {Upgrader} options.upgrader @@ -159,7 +159,7 @@ class Circuit { * * @param {any} options * @param {Function} handler - * @return {listener} + * @returns {listener} */ createListener (options, handler) { if (typeof options === 'function') { diff --git a/src/circuit/utils.js b/src/circuit/utils.js index 7426271cb7..18b61eafbb 100644 --- a/src/circuit/utils.js +++ b/src/circuit/utils.js @@ -5,8 +5,9 @@ const multihashing = require('multihashing-async') /** * Convert a namespace string into a cid. + * * @param {string} namespace - * @return {Promise} + * @returns {Promise} */ module.exports.namespaceToCid = async (namespace) => { const bytes = new TextEncoder('utf8').encode(namespace) From 3bd1768b04ea397148fb644dd876fa02ec90ead3 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 8 Oct 2020 10:42:18 +0200 Subject: [PATCH 14/17] chore: sort relay addresses to listen for public first --- package.json | 2 +- src/circuit/auto-relay.js | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 42c6c2ae87..b1ae4c5ebe 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "it-protocol-buffers": "^0.2.0", "libp2p-crypto": "^0.18.0", "libp2p-interfaces": "^0.5.1", - "libp2p-utils": "^0.2.0", + "libp2p-utils": "^0.2.1", "mafmt": "^8.0.0", "merge-options": "^2.0.0", "moving-average": "^1.0.0", diff --git a/src/circuit/auto-relay.js b/src/circuit/auto-relay.js index c0475dbada..768d17eb37 100644 --- a/src/circuit/auto-relay.js +++ b/src/circuit/auto-relay.js @@ -4,6 +4,8 @@ const debug = require('debug') const log = debug('libp2p:auto-relay') log.error = debug('libp2p:auto-relay:error') +const isPrivate = require('libp2p-utils/src/multiaddr/is-private') + const uint8ArrayFromString = require('uint8arrays/from-string') const uint8ArrayToString = require('uint8arrays/to-string') const multiaddr = require('multiaddr') @@ -131,9 +133,13 @@ class AutoRelay { let listenAddr, remoteMultiaddr, remoteAddrs try { + // Get peer known addresses and sort them per public addresses first remoteAddrs = this._peerStore.addressBook.get(connection.remotePeer) - // TODO: HOP Relays should avoid advertising private addresses! + // TODO: This sort should be customizable in the config (dialer addr sort) + remoteAddrs.sort(multiaddrsCompareFunction) + remoteMultiaddr = remoteAddrs.find(a => a.isCertified).multiaddr // Get first announced address certified + // TODO: HOP Relays should avoid advertising private addresses! } catch (_) { log.error(`${id} does not have announced certified multiaddrs`) @@ -267,4 +273,24 @@ class AutoRelay { } } +/** + * Compare function for array.sort(). + * This sort aims to move the private adresses to the end of the array. + * + * @param {Address} a + * @param {Address} b + * @returns {number} + */ +function multiaddrsCompareFunction (a, b) { + const isAPrivate = isPrivate(a.multiaddr) + const isBPrivate = isPrivate(b.multiaddr) + + if (isAPrivate && !isBPrivate) { + return 1 + } else if (!isAPrivate && isBPrivate) { + return -1 + } + return 0 +} + module.exports = AutoRelay From 558bcf9541053e597cd0e4c29a0ce8400ad475b1 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 8 Oct 2020 19:54:20 +0200 Subject: [PATCH 15/17] chore: improve logging for auto relay active listen --- src/circuit/auto-relay.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/circuit/auto-relay.js b/src/circuit/auto-relay.js index 768d17eb37..00ebc6f588 100644 --- a/src/circuit/auto-relay.js +++ b/src/circuit/auto-relay.js @@ -264,11 +264,7 @@ class AutoRelay { } } } catch (err) { - if (err.code !== 'NO_ROUTERS_AVAILABLE') { - throw err - } else { - log('there are no routers configured to find hop relay services') - } + log.error(err) } } } From 8456d0e051f8523c3b8ae5e64ce8e62b33de1671 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 27 Oct 2020 13:50:14 +0000 Subject: [PATCH 16/17] chore: store self protocols in protobook (#760) --- doc/API.md | 77 +++++++++++++------- src/identify/index.js | 23 ++++-- src/index.js | 17 ++--- src/peer-store/proto-book.js | 41 ++++++++++- test/identify/index.spec.js | 110 ++++++++++++++++------------- test/peer-store/proto-book.spec.js | 92 ++++++++++++++++++++++++ 6 files changed, 264 insertions(+), 96 deletions(-) diff --git a/doc/API.md b/doc/API.md index 69e0ca4f78..729069dd15 100644 --- a/doc/API.md +++ b/doc/API.md @@ -37,6 +37,7 @@ * [`peerStore.protoBook.add`](#peerstoreprotobookadd) * [`peerStore.protoBook.delete`](#peerstoreprotobookdelete) * [`peerStore.protoBook.get`](#peerstoreprotobookget) + * [`peerStore.protoBook.remove`](#peerstoreprotobookremove) * [`peerStore.protoBook.set`](#peerstoreprotobookset) * [`peerStore.delete`](#peerstoredelete) * [`peerStore.get`](#peerstoreget) @@ -843,32 +844,6 @@ Consider using `addressBook.add()` if you're not sure this is what you want to d peerStore.addressBook.add(peerId, multiaddr) ``` -### peerStore.protoBook.add - -Add known `protocols` of a given peer. - -`peerStore.protoBook.add(peerId, protocols)` - -#### Parameters - -| Name | Type | Description | -|------|------|-------------| -| peerId | [`PeerId`][peer-id] | peerId to set | -| protocols | `Array` | protocols to add | - -#### Returns - -| Type | Description | -|------|-------------| -| `ProtoBook` | Returns the Proto Book component | - -#### Example - -```js -peerStore.protoBook.add(peerId, protocols) -``` - - ### peerStore.keyBook.delete Delete the provided peer from the book. @@ -1091,6 +1066,31 @@ Set known metadata of a given `peerId`. peerStore.metadataBook.set(peerId, 'location', uint8ArrayFromString('Berlin')) ``` +### peerStore.protoBook.add + +Add known `protocols` of a given peer. + +`peerStore.protoBook.add(peerId, protocols)` + +#### Parameters + +| Name | Type | Description | +|------|------|-------------| +| peerId | [`PeerId`][peer-id] | peerId to set | +| protocols | `Array` | protocols to add | + +#### Returns + +| Type | Description | +|------|-------------| +| `ProtoBook` | Returns the Proto Book component | + +#### Example + +```js +peerStore.protoBook.add(peerId, protocols) +``` + ### peerStore.protoBook.delete Delete the provided peer from the book. @@ -1147,6 +1147,31 @@ peerStore.protoBook.get(peerId) // [ '/proto/1.0.0', '/proto/1.1.0' ] ``` +### peerStore.protoBook.remove + +Remove given `protocols` of a given peer. + +`peerStore.protoBook.remove(peerId, protocols)` + +#### Parameters + +| Name | Type | Description | +|------|------|-------------| +| peerId | [`PeerId`][peer-id] | peerId to set | +| protocols | `Array` | protocols to remove | + +#### Returns + +| Type | Description | +|------|-------------| +| `ProtoBook` | Returns the Proto Book component | + +#### Example + +```js +peerStore.protoBook.remove(peerId, protocols) +``` + ### peerStore.protoBook.set Set known `protocols` of a given peer. diff --git a/src/identify/index.js b/src/identify/index.js index 53c2cee8f2..351bfbe4c7 100644 --- a/src/identify/index.js +++ b/src/identify/index.js @@ -51,9 +51,8 @@ class IdentifyService { * @class * @param {object} options * @param {Libp2p} options.libp2p - * @param {Map} options.protocols - A reference to the protocols we support */ - constructor ({ libp2p, protocols }) { + constructor ({ libp2p }) { /** * @property {PeerStore} */ @@ -74,10 +73,9 @@ class IdentifyService { */ this._libp2p = libp2p - this._protocols = protocols - this.handleMessage = this.handleMessage.bind(this) + // When a new connection happens, trigger identify this.connectionManager.on('peer:connect', (connection) => { const peerId = connection.remotePeer @@ -90,6 +88,13 @@ class IdentifyService { this.pushToPeerStore() } }) + + // When self protocols change, trigger identify-push + this.peerStore.on('change:protocols', ({ peerId }) => { + if (peerId.toString() === this.peerId.toString()) { + this.pushToPeerStore() + } + }) } /** @@ -101,7 +106,7 @@ class IdentifyService { async push (connections) { const signedPeerRecord = await this.peerStore.addressBook.getRawEnvelope(this.peerId) const listenAddrs = this._libp2p.multiaddrs.map((ma) => ma.bytes) - const protocols = Array.from(this._protocols.keys()) + const protocols = this.peerStore.protoBook.get(this.peerId) || [] const pushes = connections.map(async connection => { try { @@ -132,6 +137,11 @@ class IdentifyService { * @returns {void} */ pushToPeerStore () { + // Do not try to push if libp2p node is not running + if (!this._libp2p.isStarted()) { + return + } + const connections = [] let connection for (const peer of this.peerStore.peers.values()) { @@ -251,6 +261,7 @@ class IdentifyService { } const signedPeerRecord = await this.peerStore.addressBook.getRawEnvelope(this.peerId) + const protocols = this.peerStore.protoBook.get(this.peerId) || [] const message = Message.encode({ protocolVersion: PROTOCOL_VERSION, @@ -259,7 +270,7 @@ class IdentifyService { listenAddrs: this._libp2p.multiaddrs.map((ma) => ma.bytes), signedPeerRecord, observedAddr: connection.remoteAddr.bytes, - protocols: Array.from(this._protocols.keys()) + protocols }) try { diff --git a/src/index.js b/src/index.js index f3c4b326c3..afc28fe79c 100644 --- a/src/index.js +++ b/src/index.js @@ -158,10 +158,7 @@ class Libp2p extends EventEmitter { }) // Add the identify service since we can multiplex - this.identifyService = new IdentifyService({ - libp2p: this, - protocols: this.upgrader.protocols - }) + this.identifyService = new IdentifyService({ libp2p: this }) this.handle(Object.values(IDENTIFY_PROTOCOLS), this.identifyService.handleMessage) } @@ -437,10 +434,8 @@ class Libp2p extends EventEmitter { this.upgrader.protocols.set(protocol, handler) }) - // Only push if libp2p is running - if (this.isStarted() && this.identifyService) { - this.identifyService.pushToPeerStore() - } + // Add new protocols to self protocols in the Protobook + this.peerStore.protoBook.add(this.peerId, protocols) } /** @@ -455,10 +450,8 @@ class Libp2p extends EventEmitter { this.upgrader.protocols.delete(protocol) }) - // Only push if libp2p is running - if (this.isStarted() && this.identifyService) { - this.identifyService.pushToPeerStore() - } + // Remove protocols from self protocols in the Protobook + this.peerStore.protoBook.remove(this.peerId, protocols) } async _onStarting () { diff --git a/src/peer-store/proto-book.js b/src/peer-store/proto-book.js index 073b7e47e5..a08f5a284d 100644 --- a/src/peer-store/proto-book.js +++ b/src/peer-store/proto-book.js @@ -112,13 +112,50 @@ class ProtoBook extends Book { return this } - protocols = [...newSet] - this._setData(peerId, newSet) log(`added provided protocols for ${id}`) return this } + + /** + * Removes known protocols of a provided peer. + * If the protocols did not exist before, nothing will be done. + * + * @param {PeerId} peerId + * @param {Array} protocols + * @returns {ProtoBook} + */ + remove (peerId, protocols) { + if (!PeerId.isPeerId(peerId)) { + log.error('peerId must be an instance of peer-id to store data') + throw errcode(new Error('peerId must be an instance of peer-id'), ERR_INVALID_PARAMETERS) + } + + if (!protocols) { + log.error('protocols must be provided to store data') + throw errcode(new Error('protocols must be provided'), ERR_INVALID_PARAMETERS) + } + + const id = peerId.toB58String() + const recSet = this.data.get(id) + + if (recSet) { + const newSet = new Set([ + ...recSet + ].filter((p) => !protocols.includes(p))) + + // Any protocol removed? + if (recSet.size === newSet.size) { + return this + } + + this._setData(peerId, newSet) + log(`removed provided protocols for ${id}`) + } + + return this + } } module.exports = ProtoBook diff --git a/test/identify/index.spec.js b/test/identify/index.spec.js index d3b14273b7..4a5e308c7f 100644 --- a/test/identify/index.spec.js +++ b/test/identify/index.spec.js @@ -29,18 +29,21 @@ const remoteAddr = MULTIADDRS_WEBSOCKETS[0] const listenMaddrs = [multiaddr('/ip4/127.0.0.1/tcp/15002/ws')] describe('Identify', () => { - let localPeer - let remotePeer - const protocols = new Map([ - [multicodecs.IDENTIFY, () => {}], - [multicodecs.IDENTIFY_PUSH, () => {}] - ]) + let localPeer, localPeerStore + let remotePeer, remotePeerStore + const protocols = [multicodecs.IDENTIFY, multicodecs.IDENTIFY_PUSH] before(async () => { [localPeer, remotePeer] = (await Promise.all([ PeerId.createFromJSON(Peers[0]), PeerId.createFromJSON(Peers[1]) ])) + + localPeerStore = new PeerStore({ peerId: localPeer }) + localPeerStore.protoBook.set(localPeer, protocols) + + remotePeerStore = new PeerStore({ peerId: remotePeer }) + remotePeerStore.protoBook.set(remotePeer, protocols) }) afterEach(() => { @@ -52,20 +55,19 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), - peerStore: new PeerStore({ peerId: localPeer }), - multiaddrs: listenMaddrs - }, - protocols + peerStore: localPeerStore, + multiaddrs: listenMaddrs, + isStarted: () => true + } }) - const remoteIdentify = new IdentifyService({ libp2p: { peerId: remotePeer, connectionManager: new EventEmitter(), - peerStore: new PeerStore({ peerId: remotePeer }), - multiaddrs: listenMaddrs - }, - protocols + peerStore: remotePeerStore, + multiaddrs: listenMaddrs, + isStarted: () => true + } }) const observedAddr = multiaddr('/ip4/127.0.0.1/tcp/1234') @@ -108,20 +110,20 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), - peerStore: new PeerStore({ peerId: localPeer }), - multiaddrs: listenMaddrs - }, - protocols + peerStore: localPeerStore, + multiaddrs: listenMaddrs, + isStarted: () => true + } }) const remoteIdentify = new IdentifyService({ libp2p: { peerId: remotePeer, connectionManager: new EventEmitter(), - peerStore: new PeerStore({ peerId: remotePeer }), - multiaddrs: listenMaddrs - }, - protocols + peerStore: remotePeerStore, + multiaddrs: listenMaddrs, + isStarted: () => true + } }) const observedAddr = multiaddr('/ip4/127.0.0.1/tcp/1234') @@ -167,19 +169,17 @@ describe('Identify', () => { libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), - peerStore: new PeerStore({ peerId: localPeer }), + peerStore: localPeerStore, multiaddrs: [] - }, - protocols + } }) const remoteIdentify = new IdentifyService({ libp2p: { peerId: remotePeer, connectionManager: new EventEmitter(), - peerStore: new PeerStore({ peerId: remotePeer }), + peerStore: remotePeerStore, multiaddrs: [] - }, - protocols + } }) const observedAddr = multiaddr('/ip4/127.0.0.1/tcp/1234') @@ -206,33 +206,38 @@ describe('Identify', () => { describe('push', () => { it('should be able to push identify updates to another peer', async () => { + const storedProtocols = [multicodecs.IDENTIFY, multicodecs.IDENTIFY_PUSH, '/echo/1.0.0'] const connectionManager = new EventEmitter() connectionManager.getConnection = () => { } + const localPeerStore = new PeerStore({ peerId: localPeer }) + localPeerStore.protoBook.set(localPeer, storedProtocols) + const localIdentify = new IdentifyService({ libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), - peerStore: new PeerStore({ peerId: localPeer }), - multiaddrs: listenMaddrs - }, - protocols: new Map([ - [multicodecs.IDENTIFY], - [multicodecs.IDENTIFY_PUSH], - ['/echo/1.0.0'] - ]) + peerStore: localPeerStore, + multiaddrs: listenMaddrs, + isStarted: () => true + } }) + + const remotePeerStore = new PeerStore({ peerId: remotePeer }) + remotePeerStore.protoBook.set(remotePeer, storedProtocols) + const remoteIdentify = new IdentifyService({ libp2p: { peerId: remotePeer, connectionManager, - peerStore: new PeerStore({ peerId: remotePeer }), - multiaddrs: [] + peerStore: remotePeerStore, + multiaddrs: [], + isStarted: () => true } }) // Setup peer protocols and multiaddrs - const localProtocols = new Set([multicodecs.IDENTIFY, multicodecs.IDENTIFY_PUSH, '/echo/1.0.0']) + const localProtocols = new Set(storedProtocols) const localConnectionMock = { newStream: () => { } } const remoteConnectionMock = { remotePeer: localPeer } @@ -271,33 +276,38 @@ describe('Identify', () => { // LEGACY it('should be able to push identify updates to another peer with no certified peer records support', async () => { + const storedProtocols = [multicodecs.IDENTIFY, multicodecs.IDENTIFY_PUSH, '/echo/1.0.0'] const connectionManager = new EventEmitter() connectionManager.getConnection = () => { } + const localPeerStore = new PeerStore({ peerId: localPeer }) + localPeerStore.protoBook.set(localPeer, storedProtocols) + const localIdentify = new IdentifyService({ libp2p: { peerId: localPeer, connectionManager: new EventEmitter(), - peerStore: new PeerStore({ peerId: localPeer }), - multiaddrs: listenMaddrs - }, - protocols: new Map([ - [multicodecs.IDENTIFY], - [multicodecs.IDENTIFY_PUSH], - ['/echo/1.0.0'] - ]) + peerStore: localPeerStore, + multiaddrs: listenMaddrs, + isStarted: () => true + } }) + + const remotePeerStore = new PeerStore({ peerId: remotePeer }) + remotePeerStore.protoBook.set(remotePeer, storedProtocols) + const remoteIdentify = new IdentifyService({ libp2p: { peerId: remotePeer, connectionManager, peerStore: new PeerStore({ peerId: remotePeer }), - multiaddrs: [] + multiaddrs: [], + isStarted: () => true } }) // Setup peer protocols and multiaddrs - const localProtocols = new Set([multicodecs.IDENTIFY, multicodecs.IDENTIFY_PUSH, '/echo/1.0.0']) + const localProtocols = new Set(storedProtocols) const localConnectionMock = { newStream: () => {} } const remoteConnectionMock = { remotePeer: localPeer } diff --git a/test/peer-store/proto-book.spec.js b/test/peer-store/proto-book.spec.js index 15b5199757..06dfcdf798 100644 --- a/test/peer-store/proto-book.spec.js +++ b/test/peer-store/proto-book.spec.js @@ -5,7 +5,9 @@ const chai = require('chai') chai.use(require('dirty-chai')) const { expect } = chai +const sinon = require('sinon') const pDefer = require('p-defer') +const pWaitFor = require('p-wait-for') const PeerStore = require('../../src/peer-store') @@ -224,6 +226,96 @@ describe('protoBook', () => { }) }) + describe('protoBook.remove', () => { + let peerStore, pb + + beforeEach(() => { + peerStore = new PeerStore({ peerId }) + pb = peerStore.protoBook + }) + + afterEach(() => { + peerStore.removeAllListeners() + }) + + it('throws invalid parameters error if invalid PeerId is provided', () => { + expect(() => { + pb.remove('invalid peerId') + }).to.throw(ERR_INVALID_PARAMETERS) + }) + + it('throws invalid parameters error if no protocols provided', () => { + expect(() => { + pb.remove(peerId) + }).to.throw(ERR_INVALID_PARAMETERS) + }) + + it('removes the given protocol and emits change event', async () => { + const spy = sinon.spy() + + const supportedProtocols = ['protocol1', 'protocol2'] + const removedProtocols = ['protocol1'] + const finalProtocols = supportedProtocols.filter(p => !removedProtocols.includes(p)) + + peerStore.on('change:protocols', spy) + + // Replace + pb.set(peerId, supportedProtocols) + let protocols = pb.get(peerId) + expect(protocols).to.have.deep.members(supportedProtocols) + + // Remove + pb.remove(peerId, removedProtocols) + protocols = pb.get(peerId) + expect(protocols).to.have.deep.members(finalProtocols) + + await pWaitFor(() => spy.callCount === 2) + + const [firstCallArgs] = spy.firstCall.args + const [secondCallArgs] = spy.secondCall.args + expect(arraysAreEqual(firstCallArgs.protocols, supportedProtocols)) + expect(arraysAreEqual(secondCallArgs.protocols, finalProtocols)) + }) + + it('emits on remove if the content changes', () => { + const spy = sinon.spy() + + const supportedProtocols = ['protocol1', 'protocol2'] + const removedProtocols = ['protocol2'] + const finalProtocols = supportedProtocols.filter(p => !removedProtocols.includes(p)) + + peerStore.on('change:protocols', spy) + + // set + pb.set(peerId, supportedProtocols) + + // remove (content already existing) + pb.remove(peerId, removedProtocols) + const protocols = pb.get(peerId) + expect(protocols).to.have.deep.members(finalProtocols) + + return pWaitFor(() => spy.callCount === 2) + }) + + it('does not emit on remove if the content does not change', () => { + const spy = sinon.spy() + + const supportedProtocols = ['protocol1', 'protocol2'] + const removedProtocols = ['protocol3'] + + peerStore.on('change:protocols', spy) + + // set + pb.set(peerId, supportedProtocols) + + // remove + pb.remove(peerId, removedProtocols) + + // Only one event + expect(spy.callCount).to.eql(1) + }) + }) + describe('protoBook.get', () => { let peerStore, pb From f5b6ebe1b755eb706218f4a7f385d6faf713ce90 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 9 Nov 2020 15:13:23 +0100 Subject: [PATCH 17/17] feat: rendezvous integration --- README.md | 2 + doc/CONFIGURATION.md | 63 +++++++++++++++++ package-list.json | 3 + package.json | 1 + src/config.js | 5 ++ src/index.js | 13 ++++ test/discovery/rendezvous.spec.js | 111 ++++++++++++++++++++++++++++++ test/discovery/utils.js | 41 +++++++++++ 8 files changed, 239 insertions(+) create mode 100644 test/discovery/rendezvous.spec.js create mode 100644 test/discovery/utils.js diff --git a/README.md b/README.md index e6d8af7285..db0e803dd1 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,8 @@ List of packages currently in existence for libp2p | **peer routing** | | [`libp2p-delegated-peer-routing`](//github.com/libp2p/js-libp2p-delegated-peer-routing) | [![npm](https://img.shields.io/npm/v/libp2p-delegated-peer-routing.svg?maxAge=86400&style=flat-square)](//github.com/libp2p/js-libp2p-delegated-peer-routing/releases) | [![Deps](https://david-dm.org/libp2p/js-libp2p-delegated-peer-routing.svg?style=flat-square)](https://david-dm.org/libp2p/js-libp2p-delegated-peer-routing) | [![Travis CI](https://flat.badgen.net/travis/libp2p/js-libp2p-delegated-peer-routing/master)](https://travis-ci.com/libp2p/js-libp2p-delegated-peer-routing) | [![codecov](https://codecov.io/gh/libp2p/js-libp2p-delegated-peer-routing/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/libp2p/js-libp2p-delegated-peer-routing) | [Jacob Heun](mailto:jacobheun@gmail.com) | | [`libp2p-kad-dht`](//github.com/libp2p/js-libp2p-kad-dht) | [![npm](https://img.shields.io/npm/v/libp2p-kad-dht.svg?maxAge=86400&style=flat-square)](//github.com/libp2p/js-libp2p-kad-dht/releases) | [![Deps](https://david-dm.org/libp2p/js-libp2p-kad-dht.svg?style=flat-square)](https://david-dm.org/libp2p/js-libp2p-kad-dht) | [![Travis CI](https://flat.badgen.net/travis/libp2p/js-libp2p-kad-dht/master)](https://travis-ci.com/libp2p/js-libp2p-kad-dht) | [![codecov](https://codecov.io/gh/libp2p/js-libp2p-kad-dht/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/libp2p/js-libp2p-kad-dht) | [Vasco Santos](mailto:vasco.santos@moxy.studio) | +| **service discovery** | +| [`libp2p-rendezvous`](//github.com/libp2p/js-libp2p-rendezvous) | [![npm](https://img.shields.io/npm/v/libp2p-rendezvous.svg?maxAge=86400&style=flat-square)](//github.com/libp2p/js-libp2p-rendezvous/releases) | [![Deps](https://david-dm.org/libp2p/js-libp2p-rendezvous.svg?style=flat-square)](https://david-dm.org/libp2p/js-libp2p-rendezvous) | [![Travis CI](https://flat.badgen.net/travis/libp2p/js-libp2p-rendezvous/master)](https://travis-ci.com/libp2p/js-libp2p-rendezvous) | [![codecov](https://codecov.io/gh/libp2p/js-libp2p-rendezvous/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/libp2p/js-libp2p-rendezvous) | [Vasco Santos](mailto:santos.vasco10@gmail.com) | | **utilities** | | [`libp2p-crypto`](//github.com/libp2p/js-libp2p-crypto) | [![npm](https://img.shields.io/npm/v/libp2p-crypto.svg?maxAge=86400&style=flat-square)](//github.com/libp2p/js-libp2p-crypto/releases) | [![Deps](https://david-dm.org/libp2p/js-libp2p-crypto.svg?style=flat-square)](https://david-dm.org/libp2p/js-libp2p-crypto) | [![Travis CI](https://flat.badgen.net/travis/libp2p/js-libp2p-crypto/master)](https://travis-ci.com/libp2p/js-libp2p-crypto) | [![codecov](https://codecov.io/gh/libp2p/js-libp2p-crypto/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/libp2p/js-libp2p-crypto) | [Jacob Heun](mailto:jacobheun@gmail.com) | | [`libp2p-crypto-secp256k1`](//github.com/libp2p/js-libp2p-crypto-secp256k1) | [![npm](https://img.shields.io/npm/v/libp2p-crypto-secp256k1.svg?maxAge=86400&style=flat-square)](//github.com/libp2p/js-libp2p-crypto-secp256k1/releases) | [![Deps](https://david-dm.org/libp2p/js-libp2p-crypto-secp256k1.svg?style=flat-square)](https://david-dm.org/libp2p/js-libp2p-crypto-secp256k1) | [![Travis CI](https://flat.badgen.net/travis/libp2p/js-libp2p-crypto-secp256k1/master)](https://travis-ci.com/libp2p/js-libp2p-crypto-secp256k1) | [![codecov](https://codecov.io/gh/libp2p/js-libp2p-crypto-secp256k1/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/libp2p/js-libp2p-crypto-secp256k1) | [Friedel Ziegelmayer](mailto:dignifiedquire@gmail.com) | diff --git a/doc/CONFIGURATION.md b/doc/CONFIGURATION.md index 15f34d4ff6..d82bf69a76 100644 --- a/doc/CONFIGURATION.md +++ b/doc/CONFIGURATION.md @@ -21,6 +21,7 @@ - [Setup with Content and Peer Routing](#setup-with-content-and-peer-routing) - [Setup with Relay](#setup-with-relay) - [Setup with Auto Relay](#setup-with-auto-relay) + - [Setup with Rendezvous](#setup-with-rendezvous) - [Setup with Keychain](#setup-with-keychain) - [Configuring Dialing](#configuring-dialing) - [Configuring Connection Manager](#configuring-connection-manager) @@ -457,6 +458,68 @@ const node = await Libp2p.create({ }) ``` +#### Setup with Rendezvous + +You will need to setup a rendezvous server, which will be used by rendezvous client nodes. + +A rendezvous server can be configured as follows: + +```js +const Libp2p = require('libp2p') +const TCP = require('libp2p-tcp') +const MPLEX = require('libp2p-mplex') +const { NOISE } = require('libp2p-noise') +const Rendezvous = require('libp2p-rendezvous') + +const node = await Libp2p.create({ + modules: { + transport: [TCP], + streamMuxer: [MPLEX], + connEncryption: [NOISE], + rendezvous: Rendezvous + }, + config: { + rendezvous: { // Rendezvous options (this config is part of libp2p core configurations) + server: { + enabled: true, // Allows you to be a rendezvous server for other peers + gcInterval: 3e5 // Interval for gc to check outdated rendezvous registrations + } + } + } +}) +``` + +A rendezvous client only needs the rendezvous module. However, it will need to discover and get connected with a rendezvous server. A good option is to leverage the bootstrap module for this. + +```js +const Libp2p = require('libp2p') +const TCP = require('libp2p-tcp') +const MPLEX = require('libp2p-mplex') +const { NOISE } = require('libp2p-noise') +const Rendezvous = require('libp2p-rendezvous') +const Bootstrap = require('libp2p-bootstrap') + +const node = await Libp2p.create({ + modules: { + transport: [TCP], + streamMuxer: [MPLEX], + connEncryption: [NOISE], + rendezvous: Rendezvous, + peerDiscovery: [Bootstrap] + }, + config: { + peerDiscovery: { + bootstrap: { + enabled: true, + list: [ + // Insert rendezvous servers multiaddrs + ] + } + } + } +}) +``` + #### Setup with Keychain Libp2p allows you to setup a secure keychain to manage your keys. The keychain configuration object should have the following properties: diff --git a/package-list.json b/package-list.json index 84c648a971..7612cc4b27 100644 --- a/package-list.json +++ b/package-list.json @@ -43,6 +43,9 @@ ["libp2p/js-libp2p-delegated-peer-routing", "libp2p-delegated-peer-routing"], ["libp2p/js-libp2p-kad-dht", "libp2p-kad-dht"], + "service discovery", + ["libp2p/js-libp2p-rendezvous", "libp2p-rendezvous"], + "utilities", ["libp2p/js-libp2p-crypto", "libp2p-crypto"], ["libp2p/js-libp2p-crypto-secp256k1", "libp2p-crypto-secp256k1"], diff --git a/package.json b/package.json index b1ae4c5ebe..92dd25c253 100644 --- a/package.json +++ b/package.json @@ -108,6 +108,7 @@ "libp2p-mdns": "^0.15.0", "libp2p-mplex": "^0.10.1", "libp2p-noise": "^2.0.0", + "libp2p-rendezvous": "libp2p/js-libp2p-rendezvous#feat/rendezvous-protocol-full-implementation", "libp2p-secio": "^0.13.1", "libp2p-tcp": "^0.15.1", "libp2p-webrtc-star": "^0.20.0", diff --git a/src/config.js b/src/config.js index 2337e249ae..380218486e 100644 --- a/src/config.js +++ b/src/config.js @@ -71,6 +71,11 @@ const DefaultConfig = { maxListeners: 2 } }, + rendezvous: { + server: { + enabled: false + } + }, transport: {} } } diff --git a/src/index.js b/src/index.js index afc28fe79c..4390af6572 100644 --- a/src/index.js +++ b/src/index.js @@ -190,6 +190,15 @@ class Libp2p extends EventEmitter { this.pubsub = PubsubAdapter(Pubsub, this, this._config.pubsub) } + // Create rendezvous if provided + if (this._modules.rendezvous) { + const Rendezvous = this._modules.rendezvous + this.rendezvous = new Rendezvous({ + libp2p: this, + ...this._config.rendezvous + }) + } + // Attach remaining APIs // peer and content routing will automatically get modules from _modules and _dht this.peerRouting = peerRouting(this) @@ -269,6 +278,8 @@ class Libp2p extends EventEmitter { this.metrics && this.metrics.stop() ]) + this.rendezvous && this.rendezvous.stop() + await this.transportManager.close() ping.unmount(this) @@ -500,6 +511,8 @@ class Libp2p extends EventEmitter { this.connectionManager.start() + this.rendezvous && this.rendezvous.start() + // Peer discovery await this._setupPeerDiscovery() diff --git a/test/discovery/rendezvous.spec.js b/test/discovery/rendezvous.spec.js new file mode 100644 index 0000000000..53a8ee7fc0 --- /dev/null +++ b/test/discovery/rendezvous.spec.js @@ -0,0 +1,111 @@ +'use strict' +/* eslint-env mocha */ + +const { expect } = require('aegir/utils/chai') +const pWaitFor = require('p-wait-for') + +const Envelope = require('../../src/record/envelope') +const PeerRecord = require('../../src/record/peer-record') + +const { + rendezvousClientOptions, + rendezvousServerOptions, + listenAddrs +} = require('./utils') +const peerUtils = require('../utils/creators/peer') + +describe('libp2p.rendezvous', () => { + let libp2p, remoteLibp2p, rendezvousLibp2p + + // Create Rendezvous server node + before(async () => { + [rendezvousLibp2p] = await peerUtils.createPeer({ + number: 1, + fixture: false, + config: { + ...rendezvousServerOptions, + addresses: { + listen: listenAddrs + } + } + }) + }) + + // Create libp2p nodes to act as rendezvous clients + before(async () => { + [libp2p, remoteLibp2p] = await peerUtils.createPeer({ + number: 2, + populateAddressBooks: false, + config: { + ...rendezvousClientOptions, + addresses: { + listen: listenAddrs + }, + config: { + peerDiscovery: { + bootstrap: { // Bootstrap rendezvous server + enabled: true, + list: [ + `${rendezvousLibp2p.multiaddrs[0]}/p2p/${rendezvousLibp2p.peerId.toB58String()}` + ] + } + } + } + } + }) + }) + + // Wait for bootstrap peer connected and identified as rendezvous server + before(async () => { + await pWaitFor(() => Boolean(rendezvousLibp2p.connectionManager.get(libp2p.peerId)) && + Boolean(rendezvousLibp2p.connectionManager.get(remoteLibp2p.peerId)) + ) + + await pWaitFor(() => libp2p.rendezvous._rendezvousPoints.size === 1 && + remoteLibp2p.rendezvous._rendezvousPoints.size === 1 + ) + }) + + after(() => { + return Promise.all([libp2p, remoteLibp2p, rendezvousLibp2p].map(node => node.stop())) + }) + + it('should have rendezvous libp2p node as rendezvous server', () => { + expect(libp2p.rendezvous._rendezvousPoints.get(rendezvousLibp2p.peerId.toB58String())).to.exist() + expect(remoteLibp2p.rendezvous._rendezvousPoints.get(rendezvousLibp2p.peerId.toB58String())).to.exist() + }) + + it('should discover remoteLibp2p when it registers on a namespace', async () => { + const namespace = '/test-namespace' + const registers = [] + + // libp2p does not discovery any peer registered + for await (const reg of libp2p.rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } + + // remoteLibp2p register itself on namespace + await remoteLibp2p.rendezvous.register(namespace) + + // libp2p discover remote libp2p + for await (const reg of libp2p.rendezvous.discover(namespace)) { // eslint-disable-line + registers.push(reg) + } + + expect(registers).to.have.lengthOf(1) + expect(registers[0].signedPeerRecord).to.exist() + expect(registers[0].ns).to.eql(namespace) + + // Validate peer + const envelope = await Envelope.openAndCertify(registers[0].signedPeerRecord, PeerRecord.DOMAIN) + expect(envelope.peerId.equals(remoteLibp2p.peerId)).to.eql(true) + + // Validate multiaddrs + const rec = PeerRecord.createFromProtobuf(envelope.payload) + expect(rec.multiaddrs.length).to.eql(remoteLibp2p.multiaddrs.length) + + rec.multiaddrs.forEach((ma, index) => { + expect(ma).to.eql(remoteLibp2p.multiaddrs[index]) + }) + }) +}) diff --git a/test/discovery/utils.js b/test/discovery/utils.js new file mode 100644 index 0000000000..37ffb3ed57 --- /dev/null +++ b/test/discovery/utils.js @@ -0,0 +1,41 @@ +'use strict' + +const Bootstrap = require('libp2p-bootstrap') +const Rendezvous = require('libp2p-rendezvous') + +const mergeOptions = require('merge-options') +const { isNode } = require('ipfs-utils/src/env') +const baseOptions = require('../utils/base-options.browser') +const { MULTIADDRS_WEBSOCKETS } = require('../fixtures/browser') + +module.exports.baseOptions = baseOptions + +module.exports.listenAddrs = isNode + ? ['/ip4/127.0.0.1/tcp/0/ws'] : [`${MULTIADDRS_WEBSOCKETS[0]}/p2p-circuit`] + +module.exports.rendezvousClientOptions = mergeOptions(baseOptions, { + modules: { + rendezvous: Rendezvous, + peerDiscovery: [Bootstrap] + }, + config: { + rendezvous: { + server: { + enabled: false + } + } + } +}) + +module.exports.rendezvousServerOptions = mergeOptions(baseOptions, { + modules: { + rendezvous: Rendezvous + }, + config: { + rendezvous: { + server: { + enabled: true + } + } + } +})