feat: address and proto books (#590)

* feat: address and proto books

* chore: apply suggestions from code review

Co-Authored-By: Jacob Heun <jacobheun@gmail.com>

* chore: minor fixes and initial tests added

* chore: integrate new peer-store with code using adapters for other modules

* chore: do not use peerstore.put on get-peer-info

* chore: apply suggestions from code review

Co-Authored-By: Jacob Heun <jacobheun@gmail.com>

* chore: add new peer store tests

* chore: apply suggestions from code review

Co-Authored-By: Jacob Heun <jacobheun@gmail.com>

Co-authored-by: Jacob Heun <jacobheun@gmail.com>

Author: vasco-santos

doc/API.md

@@ -5,7 +5,6 @@ const errCode = require('err-code')
 const TimeoutController = require('timeout-abort-controller')
 const anySignal = require('any-signal')
 const PeerId = require('peer-id')
-const PeerInfo = require('peer-info')
 const debug = require('debug')
 const log = debug('libp2p:dialer')
 log.error = debug('libp2p:dialer:error')
@@ -62,13 +61,13 @@ class Dialer {
    * The dial to the first address that is successfully able to upgrade a connection
    * will be used.
    *
-   * @param {PeerInfo|Multiaddr} peer The peer to dial
+   * @param {PeerId|Multiaddr} peerId The peer to dial
    * @param {object} [options]
    * @param {AbortSignal} [options.signal] An AbortController signal
    * @returns {Promise<Connection>}
    */
-  async connectToPeer (peer, options = {}) {
-    const dialTarget = this._createDialTarget(peer)
+  async connectToPeer (peerId, options = {}) {
+    const dialTarget = this._createDialTarget(peerId)
     if (dialTarget.addrs.length === 0) {
       throw errCode(new Error('The dial request has no addresses'), codes.ERR_NO_VALID_ADDRESSES)
     }
@@ -100,7 +99,7 @@ class Dialer {
    * Creates a DialTarget. The DialTarget is used to create and track
    * the DialRequest to a given peer.
    * @private
-   * @param {PeerInfo|Multiaddr} peer A PeerId or Multiaddr
+   * @param {PeerId|Multiaddr} peer A PeerId or Multiaddr
    * @returns {DialTarget}
    */
   _createDialTarget (peer) {
@@ -111,7 +110,10 @@ class Dialer {
         addrs: [dialable]
       }
     }
-    const addrs = this.peerStore.multiaddrsForPeer(dialable)
+
+    dialable.multiaddrs && this.peerStore.addressBook.add(dialable.id, Array.from(dialable.multiaddrs))
+    const addrs = this.peerStore.addressBook.getMultiaddrsForPeer(dialable.id)
+
     return {
       id: dialable.id.toB58String(),
       addrs
@@ -179,21 +181,27 @@ class Dialer {
     this.tokens.push(token)
   }
 
+  /**
+   * PeerInfo object
+   * @typedef {Object} peerInfo
+   * @property {Multiaddr} multiaddr peer multiaddr.
+   * @property {PeerId} id peer id.
+   */
+
   /**
    * Converts the given `peer` into a `PeerInfo` or `Multiaddr`.
    * @static
-   * @param {PeerInfo|PeerId|Multiaddr|string} peer
-   * @returns {PeerInfo|Multiaddr}
+   * @param {PeerId|Multiaddr|string} peer
+   * @returns {peerInfo|Multiaddr}
    */
   static getDialable (peer) {
-    if (PeerInfo.isPeerInfo(peer)) return peer
     if (typeof peer === 'string') {
       peer = multiaddr(peer)
     }
 
-    let addr
+    let addrs
     if (multiaddr.isMultiaddr(peer)) {
-      addr = peer
+      addrs = new Set([peer]) // TODO: after peer-info removal, a Set should not be needed
       try {
         peer = PeerId.createFromCID(peer.getPeerId())
       } catch (err) {
@@ -202,10 +210,12 @@ class Dialer {
     }
 
     if (PeerId.isPeerId(peer)) {
-      peer = new PeerInfo(peer)
+      peer = {
+        id: peer,
+        multiaddrs: addrs
+      }
     }
 
-    addr && peer.multiaddrs.add(addr)
     return peer
   }
 }

src/dialer/index.js

@@ -38,7 +38,12 @@ function getPeerInfo (peer, peerStore) {
 
   addr && peer.multiaddrs.add(addr)
 
-  return peerStore ? peerStore.put(peer) : peer
+  if (peerStore) {
+    peerStore.addressBook.add(peer.id, peer.multiaddrs.toArray())
+    peerStore.protoBook.add(peer.id, Array.from(peer.protocols))
+  }
+
+  return peer
 }
 
 /**

src/get-peer-info.js

@@ -7,7 +7,6 @@ const lp = require('it-length-prefixed')
 const pipe = require('it-pipe')
 const { collect, take, consume } = require('streaming-iterables')
 
-const PeerInfo = require('peer-info')
 const PeerId = require('peer-id')
 const multiaddr = require('multiaddr')
 const { toBuffer } = require('it-buffer')
@@ -28,39 +27,6 @@ const errCode = require('err-code')
 const { codes } = require('../errors')
 
 class IdentifyService {
-  /**
-   * Replaces the multiaddrs on the given `peerInfo`,
-   * with the provided `multiaddrs`
-   * @param {PeerInfo} peerInfo
-   * @param {Array<Multiaddr>|Array<Buffer>} multiaddrs
-   */
-  static updatePeerAddresses (peerInfo, multiaddrs) {
-    if (multiaddrs && multiaddrs.length > 0) {
-      peerInfo.multiaddrs.clear()
-      multiaddrs.forEach(ma => {
-        try {
-          peerInfo.multiaddrs.add(ma)
-        } catch (err) {
-          log.error('could not add multiaddr', err)
-        }
-      })
-    }
-  }
-
-  /**
-   * Replaces the protocols on the given `peerInfo`,
-   * with the provided `protocols`
-   * @static
-   * @param {PeerInfo} peerInfo
-   * @param {Array<string>} protocols
-   */
-  static updatePeerProtocols (peerInfo, protocols) {
-    if (protocols && protocols.length > 0) {
-      peerInfo.protocols.clear()
-      protocols.forEach(proto => peerInfo.protocols.add(proto))
-    }
-  }
-
   /**
    * Takes the `addr` and converts it to a Multiaddr if possible
    * @param {Buffer|String} addr
@@ -182,19 +148,18 @@ class IdentifyService {
     } = message
 
     const id = await PeerId.createFromPubKey(publicKey)
-    const peerInfo = new PeerInfo(id)
+
     if (connection.remotePeer.toB58String() !== id.toB58String()) {
       throw errCode(new Error('identified peer does not match the expected peer'), codes.ERR_INVALID_PEER)
     }
 
     // Get the observedAddr if there is one
     observedAddr = IdentifyService.getCleanMultiaddr(observedAddr)
 
-    // Copy the listenAddrs and protocols
-    IdentifyService.updatePeerAddresses(peerInfo, listenAddrs)
-    IdentifyService.updatePeerProtocols(peerInfo, protocols)
+    // Update peers data in PeerStore
+    this.registrar.peerStore.addressBook.set(id, listenAddrs.map((addr) => multiaddr(addr)))
+    this.registrar.peerStore.protoBook.set(id, protocols)
 
-    this.registrar.peerStore.replace(peerInfo)
     // TODO: Track our observed address so that we can score it
     log('received observed address of %s', observedAddr)
   }
@@ -274,20 +239,16 @@ class IdentifyService {
       return log.error('received invalid message', err)
     }
 
-    // Update the listen addresses
-    const peerInfo = new PeerInfo(connection.remotePeer)
-
+    // Update peers data in PeerStore
+    const id = connection.remotePeer
     try {
-      IdentifyService.updatePeerAddresses(peerInfo, message.listenAddrs)
+      this.registrar.peerStore.addressBook.set(id, message.listenAddrs.map((addr) => multiaddr(addr)))
     } catch (err) {
       return log.error('received invalid listen addrs', err)
     }
 
     // Update the protocols
-    IdentifyService.updatePeerProtocols(peerInfo, message.protocols)
-
-    // Update the peer in the PeerStore
-    this.registrar.peerStore.replace(peerInfo)
+    this.registrar.peerStore.protoBook.set(id, message.protocols)
   }
 }
 

src/identify/index.js

@@ -60,7 +60,7 @@ class Libp2p extends EventEmitter {
       localPeer: this.peerInfo.id,
       metrics: this.metrics,
       onConnection: (connection) => {
-        const peerInfo = this.peerStore.put(new PeerInfo(connection.remotePeer), { silent: true })
+        const peerInfo = new PeerInfo(connection.remotePeer)
         this.registrar.onConnect(peerInfo, connection)
         this.connectionManager.onConnect(connection)
         this.emit('peer:connect', peerInfo)
@@ -292,7 +292,11 @@ class Libp2p extends EventEmitter {
     const dialable = Dialer.getDialable(peer)
     let connection
     if (PeerInfo.isPeerInfo(dialable)) {
-      this.peerStore.put(dialable, { silent: true })
+      // TODO Inconsistency from: getDialable adds a set, while regular peerInfo uses a Multiaddr set
+      // This should be handled on `peer-info` removal
+      const multiaddrs = dialable.multiaddrs.toArray ? dialable.multiaddrs.toArray() : Array.from(dialable.multiaddrs)
+      this.peerStore.addressBook.add(dialable.id, multiaddrs)
+
       connection = this.registrar.getConnection(dialable)
     }
 
@@ -338,7 +342,7 @@ class Libp2p extends EventEmitter {
   async ping (peer) {
     const peerInfo = await getPeerInfo(peer, this.peerStore)
 
-    return ping(this, peerInfo)
+    return ping(this, peerInfo.id)
   }
 
   /**
@@ -440,7 +444,10 @@ class Libp2p extends EventEmitter {
       log.error(new Error(codes.ERR_DISCOVERED_SELF))
       return
     }
-    this.peerStore.put(peerInfo)
+
+    // TODO: once we deprecate peer-info, we should only set if we have data
+    this.peerStore.addressBook.add(peerInfo.id, peerInfo.multiaddrs.toArray())
+    this.peerStore.protoBook.set(peerInfo.id, Array.from(peerInfo.protocols))
   }
 
   /**

src/index.js

@@ -1,3 +1,89 @@
-# Peerstore
+# PeerStore
 
-WIP
+Libp2p's PeerStore is responsible for keeping an updated register with the relevant information of the known peers. It should be the single source of truth for all peer data, where a subsystem can learn about peers' data and where someone can listen for updates. The PeerStore comprises four main components: `addressBook`, `keyBook`, `protocolBook` and `metadataBook`.
+
+The PeerStore manages the high level operations on its inner books. Moreover, the PeerStore should be responsible for notifying interested parties of relevant events, through its Event Emitter.
+
+## Data gathering
+
+Several libp2p subsystems will perform operations, which will gather relevant information about peers. Some operations might not have this as an end goal, but can also gather important data.
+
+In a libp2p node's life, it will discover peers through its discovery protocols. In a typical discovery protocol, addresses of the peer are discovered along with its peer id. Once this happens, the PeerStore should collect this information for future (or immediate) usage by other subsystems. When the information is stored, the PeerStore should inform interested parties of the peer discovered (`peer` event).
+
+Taking into account a different scenario, a peer might perform/receive a dial request to/from a unkwown peer. In such a scenario, the PeerStore must store the peer's multiaddr once a connection is established. 
+
+After a connection is established with a peer, the Identify protocol will run automatically. A stream is created and peers exchange their information (Multiaddrs, running protocols and their public key). Once this information is obtained, it should be added to the PeerStore. In this specific case, as we are speaking to the source of truth, we should ensure the PeerStore is prioritizing these records. If the recorded `multiaddrs` or `protocols` have changed, interested parties must be informed via the `change:multiaddrs` or `change:protocols` events respectively.
+
+In the background, the Identify Service is also waiting for protocol change notifications of peers via the IdentifyPush protocol. Peers may leverage the `identify-push` message to communicate protocol changes to all connected peers, so that their PeerStore can be updated with the updated protocols. As the `identify-push` also sends complete and updated information, the data in the PeerStore can be replaced.
+
+(To consider: Should we not replace until we get to multiaddr confidence? we might loose true information as we will talk with older nodes on the network.)
+
+While it is currently not supported in js-libp2p, future iterations may also support the [IdentifyDelta protocol](https://github.com/libp2p/specs/pull/176).
+
+It is also possible to gather relevant information for peers from other protocols / subsystems. For instance, in `DHT` operations, nodes can exchange peer data as part of the `DHT` operation. In this case, we can learn additional information about a peer we already know. In this scenario the PeerStore should not replace the existing data it has, just add it.
+
+## Data Consumption
+
+When the PeerStore data is updated, this information might be important for different parties.
+
+Every time a peer needs to dial another peer, it is essential that it knows the multiaddrs used by the peer, in order to perform a successful dial to it. The same is true for pinging a peer. While the `AddressBook` is going to keep its data updated, it will also emit `change:multiaddrs` events so that subsystems/users interested in knowing these changes can be notified instead of polling the `AddressBook`.
+
+Everytime a peer starts/stops supporting a protocol, libp2p subsystems or users might need to act accordingly. `js-libp2p` registrar orchestrates known peers, established connections and protocol topologies. This way, once a protocol is supported for a peer, the topology of that protocol should be informed that a new peer may be used and the subsystem can decide if it should open a new stream with that peer or not. For these situations, the `ProtoBook` will emit `change:protocols` events whenever supported protocols of a peer change.
+
+## PeerStore implementation
+
+The PeerStore wraps four main components: `addressBook`, `keyBook`, `protocolBook` and `metadataBook`. Moreover, it provides a high level API for those components, as well as data events.
+
+### Components
+
+#### Address Book
+
+The `addressBook` keeps the known multiaddrs of a peer. The multiaddrs of each peer may change over time and the Address Book must account for this.
+
+`Map<string, multiaddrInfo>`
+
+A `peerId.toString()` identifier mapping to a `multiaddrInfo` object, which should have the following structure:
+
+```js
+{
+  multiaddr: <Multiaddr>
+}
+```
+
+#### Key Book
+
+The `keyBook` tracks the keys of the peers.
+
+**Not Yet Implemented**
+
+#### Protocol Book
+
+The `protoBook` holds the identifiers of the protocols supported by each peer. The protocols supported by each peer are dynamic and will change over time.
+
+`Map<string, Set<string>>`
+
+A `peerId.toString()` identifier mapping to a `Set` of protocol identifier strings.
+
+#### Metadata Book
+
+**Not Yet Implemented**
+
+### API
+
+For the complete API documentation, you should check the [API.md](../../doc/API.md).
+
+Access to its underlying books:
+
+- `peerStore.protoBook.*`
+- `peerStore.addressBook.*`
+
+### Events
+
+- `peer` - emitted when a new peer is added.
+- `change:multiaadrs` - emitted when a known peer has a different set of multiaddrs.
+- `change:protocols` - emitted when a known peer supports a different set of protocols.
+
+## Future Considerations
+
+- If multiaddr TTLs are added, the PeerStore may schedule jobs to delete all addresses that exceed the TTL to prevent AddressBook bloating
+- Further API methods will probably need to be added in the context of multiaddr validity and confidence.