From 58900a57d7eb07b2ddbc83ccdc7e1abdd810844b Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Fri, 27 Jul 2018 18:01:39 +0200 Subject: [PATCH 01/19] feat: allow for configuring content and peer routing --- package.json | 2 ++ src/config.js | 8 +++++--- src/index.js | 5 +++-- test/config.spec.js | 25 +++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e9f27741de..d507d99808 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,8 @@ "dirty-chai": "^2.0.1", "electron-webrtc": "~0.3.0", "libp2p-circuit": "~0.2.1", + "libp2p-delegated-content-routing": "github:libp2p/js-libp2p-delegated-content-routing#master", + "libp2p-delegated-peer-routing": "github:libp2p/js-libp2p-delegated-peer-routing#master", "libp2p-kad-dht": "~0.10.5", "libp2p-mdns": "~0.12.0", "libp2p-mplex": "~0.8.2", diff --git a/src/config.js b/src/config.js index 7cb9def5ec..2b7485254a 100644 --- a/src/config.js +++ b/src/config.js @@ -10,14 +10,16 @@ const OptionsSchema = Joi.object({ peerInfo: Joi.object().required(), peerBook: Joi.object(), modules: Joi.object().keys({ - transport: Joi.array().items(ModuleSchema).min(1).required(), - streamMuxer: Joi.array().items(ModuleSchema).allow(null), connEncryption: Joi.array().items(ModuleSchema).allow(null), connProtector: Joi.object().keys({ protect: Joi.func().required() }).unknown(), + contentRouting: Joi.object(), + dht: ModuleSchema.allow(null), peerDiscovery: Joi.array().items(ModuleSchema).allow(null), - dht: ModuleSchema.allow(null) + peerRouting: Joi.object(), + streamMuxer: Joi.array().items(ModuleSchema).allow(null), + transport: Joi.array().items(ModuleSchema).min(1).required() }).required(), config: Joi.object().keys({ peerDiscovery: Joi.object().allow(null), diff --git a/src/index.js b/src/index.js index 12056fd948..91a2c74309 100644 --- a/src/index.js +++ b/src/index.js @@ -102,8 +102,9 @@ class Node extends EventEmitter { } // Attach remaining APIs - this.peerRouting = peerRouting(this) - this.contentRouting = contentRouting(this) + // If peer or content routing modules have been provided, use those, otherwise use the dht + this.peerRouting = this._modules.peerRouting || peerRouting(this) + this.contentRouting = this._modules.contentRouting || contentRouting(this) this.dht = dht(this) this._getPeerInfo = getPeerInfo(this) diff --git a/test/config.spec.js b/test/config.spec.js index 892cccce1d..60b8ca182f 100644 --- a/test/config.spec.js +++ b/test/config.spec.js @@ -9,6 +9,8 @@ const PeerId = require('peer-id') const waterfall = require('async/waterfall') const WS = require('libp2p-websockets') const Bootstrap = require('libp2p-bootstrap') +const DelegatedPeerRouter = require('libp2p-delegated-peer-routing') +const DelegatedContentRouter = require('libp2p-delegated-content-routing') const validateConfig = require('../src/config').validate @@ -98,6 +100,29 @@ describe('configuration', () => { expect(validateConfig(options)).to.deep.equal(expected) }) + it('should allow for delegated content and peer routing', () => { + const peerRouter = new DelegatedPeerRouter() + const contentRouter = new DelegatedContentRouter(peerInfo) + + const options = { + peerInfo, + modules: { + transport: [ WS ], + peerDiscovery: [ Bootstrap ], + peerRouting: peerRouter, + contentRouting: contentRouter + }, + config: { + peerDiscovery: { + bootstrap: { + interval: 1000, + enabled: true + } + } + } + } + }) + it('should not allow for dht to be enabled without it being provided', () => { const options = { peerInfo, From 870d675dd168fe172831320968026c5acd3a4e37 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Fri, 27 Jul 2018 18:20:13 +0200 Subject: [PATCH 02/19] test: clean up the peer routing tests --- test/peer-routing.node.js | 127 +++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 69 deletions(-) diff --git a/test/peer-routing.node.js b/test/peer-routing.node.js index b232e423ca..9dd4fadcca 100644 --- a/test/peer-routing.node.js +++ b/test/peer-routing.node.js @@ -12,87 +12,76 @@ const _times = require('lodash.times') const createNode = require('./utils/create-node') describe('.peerRouting', () => { - let nodeA - let nodeB - let nodeC - let nodeD - let nodeE + describe('via the dht', () => { + let nodeA + let nodeB + let nodeC + let nodeD + let nodeE - before(function (done) { - this.timeout(5 * 1000) + before('create the outer ring of connections', function (done) { + this.timeout(5 * 1000) - const tasks = _times(5, () => (cb) => { - createNode('/ip4/0.0.0.0/tcp/0', { - config: { - EXPERIMENTAL: { - dht: true + const tasks = _times(5, () => (cb) => { + createNode('/ip4/0.0.0.0/tcp/0', { + config: { + EXPERIMENTAL: { + dht: true + } } - } - }, (err, node) => { - expect(err).to.not.exist() - node.start((err) => cb(err, node)) + }, (err, node) => { + expect(err).to.not.exist() + node.start((err) => cb(err, node)) + }) }) - }) - - parallel(tasks, (err, nodes) => { - expect(err).to.not.exist() - nodeA = nodes[0] - nodeB = nodes[1] - nodeC = nodes[2] - nodeD = nodes[3] - nodeE = nodes[4] - - parallel([ - (cb) => nodeA.dial(nodeB.peerInfo, cb), - (cb) => nodeB.dial(nodeC.peerInfo, cb), - (cb) => nodeC.dial(nodeD.peerInfo, cb), - (cb) => nodeD.dial(nodeE.peerInfo, cb), - (cb) => nodeE.dial(nodeA.peerInfo, cb) - ], done) - }) - }) - - after((done) => { - parallel([ - (cb) => nodeA.stop(cb), - (cb) => nodeB.stop(cb), - (cb) => nodeC.stop(cb), - (cb) => nodeD.stop(cb), - (cb) => nodeE.stop(cb) - ], done) - }) - describe('el ring', () => { - it('let kbucket get filled', (done) => { - setTimeout(() => done(), 250) - }) - - it('nodeA.dial by Id to node C', (done) => { - nodeA.dial(nodeC.peerInfo.id, (err) => { + parallel(tasks, (err, nodes) => { expect(err).to.not.exist() - done() - }) - }) + nodeA = nodes[0] + nodeB = nodes[1] + nodeC = nodes[2] + nodeD = nodes[3] + nodeE = nodes[4] - it('nodeB.dial by Id to node D', (done) => { - nodeB.dial(nodeD.peerInfo.id, (err) => { - expect(err).to.not.exist() - done() + parallel([ + (cb) => nodeA.dial(nodeB.peerInfo, cb), + (cb) => nodeB.dial(nodeC.peerInfo, cb), + (cb) => nodeC.dial(nodeD.peerInfo, cb), + (cb) => nodeD.dial(nodeE.peerInfo, cb), + (cb) => nodeE.dial(nodeA.peerInfo, cb) + ], (err) => { + expect(err).to.not.exist() + // Give the kbucket time to fill in the dht + setTimeout(done, 250) + }) }) }) - it('nodeC.dial by Id to node E', (done) => { - nodeC.dial(nodeE.peerInfo.id, (err) => { - expect(err).to.not.exist() - done() - }) + after((done) => { + parallel([ + (cb) => nodeA.stop(cb), + (cb) => nodeB.stop(cb), + (cb) => nodeC.stop(cb), + (cb) => nodeD.stop(cb), + (cb) => nodeE.stop(cb) + ], done) }) - it('nodeB.peerRouting.findPeer(nodeE.peerInfo.id)', (done) => { - nodeB.peerRouting.findPeer(nodeE.peerInfo.id, (err, peerInfo) => { - expect(err).to.not.exist() - expect(nodeE.peerInfo.id.toB58String()).to.equal(peerInfo.id.toB58String()) - done() + describe('connected in an el ring', () => { + it('should be able to find a peer we are not directly connected to', (done) => { + parallel([ + (cb) => nodeA.dial(nodeC.peerInfo.id, cb), + (cb) => nodeB.dial(nodeD.peerInfo.id, cb), + (cb) => nodeC.dial(nodeE.peerInfo.id, cb) + ], (err) => { + if (err) throw err + expect(err).to.not.exist() + nodeB.peerRouting.findPeer(nodeE.peerInfo.id, (err, peerInfo) => { + expect(err).to.not.exist() + expect(nodeE.peerInfo.id.toB58String()).to.equal(peerInfo.id.toB58String()) + done() + }) + }) }) }) }) From 50b30a0a142b7b4634d336766cff33f1ac780e0e Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Fri, 17 Aug 2018 16:00:02 +0200 Subject: [PATCH 03/19] test: add tests for delegate peer routing --- package.json | 1 + test/peer-routing.node.js | 119 +++++++++++++++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index d507d99808..b11cff0622 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "libp2p-websockets": "~0.12.0", "mafmt": "^6.0.2", "multiaddr": "^5.0.0", + "nock": "^9.4.3", "peer-book": "~0.8.0", "peer-id": "~0.11.0", "peer-info": "~0.14.1" diff --git a/test/peer-routing.node.js b/test/peer-routing.node.js index 9dd4fadcca..a3230740bc 100644 --- a/test/peer-routing.node.js +++ b/test/peer-routing.node.js @@ -8,7 +8,11 @@ chai.use(require('dirty-chai')) const expect = chai.expect const parallel = require('async/parallel') const _times = require('lodash.times') +const DelegatedPeerRouter = require('libp2p-delegated-peer-routing') +const sinon = require('sinon') +const nock = require('nock') +const jsonPeerId = require('./fixtures/test-peer.json') const createNode = require('./utils/create-node') describe('.peerRouting', () => { @@ -19,9 +23,7 @@ describe('.peerRouting', () => { let nodeD let nodeE - before('create the outer ring of connections', function (done) { - this.timeout(5 * 1000) - + before('create the outer ring of connections', (done) => { const tasks = _times(5, () => (cb) => { createNode('/ip4/0.0.0.0/tcp/0', { config: { @@ -67,6 +69,15 @@ describe('.peerRouting', () => { ], done) }) + it('should use the nodes dht', (done) => { + const stub = sinon.stub(nodeA._dht, 'findPeer').callsFake(() => { + stub.restore() + done() + }) + + nodeA.peerRouting.findPeer() + }) + describe('connected in an el ring', () => { it('should be able to find a peer we are not directly connected to', (done) => { parallel([ @@ -85,4 +96,106 @@ describe('.peerRouting', () => { }) }) }) + + describe('via a delegate', () => { + let nodeA + let delegate + + before((done) => { + parallel([ + // Create the node using the delegate + (cb) => { + delegate = new DelegatedPeerRouter({ + host: 'ipfs.io', + protocol: 'https', + port: '443' + }) + createNode('/ip4/0.0.0.0/tcp/0', { + modules: { + peerRouting: delegate + }, + config: { + EXPERIMENTAL: { + dht: true + } + } + }, (err, node) => { + expect(err).to.not.exist() + nodeA = node + nodeA.start(cb) + }) + } + ], done) + }) + + afterEach(nock.cleanAll) + + it('should use the delegate router to find peers', (done) => { + const stub = sinon.stub(delegate, 'findPeer').callsFake(() => { + stub.restore() + done() + }) + nodeA.peerRouting.findPeer() + }) + + it('should be able to find a peer', (done) => { + const peerKey = 'key of a peer on the network' + const mockApi = nock('https://ipfs.io') + .post('/api/v0/dht/findpeer') + .query({ + 'arg': peerKey, + 'stream-channels': true + }) + .reply(200, `{"Extra":"","ID":"some other id","Responses":null,"Type":0}\n{"Extra":"","ID":"","Responses":[{"Addrs":["/ip4/127.0.0.1/tcp/4001"],"ID":"${peerKey}"}],"Type":2}\n`, [ + 'Content-Type', 'application/json', + 'X-Chunked-Output', '1' + ]) + + nodeA.peerRouting.findPeer(peerKey, (err, peerInfo) => { + expect(err).to.not.exist() + expect(peerInfo.id).to.equal(peerKey) + expect(mockApi.isDone()).to.equal(true) + done() + }) + }) + + it('should error when a peer cannot be found', (done) => { + const peerKey = 'key of a peer not on the network' + const mockApi = nock('https://ipfs.io') + .post('/api/v0/dht/findpeer') + .query({ + 'arg': peerKey, + 'stream-channels': true + }) + .reply(200, `{"Extra":"","ID":"some other id","Responses":null,"Type":6}\n{"Extra":"","ID":"yet another id","Responses":null,"Type":0}\n{"Extra":"routing:not found","ID":"","Responses":null,"Type":3}\n`, [ + 'Content-Type', 'application/json', + 'X-Chunked-Output', '1' + ]) + + nodeA.peerRouting.findPeer(peerKey, (err, peerInfo) => { + expect(err).to.exist() + expect(peerInfo).to.not.exist() + expect(mockApi.isDone()).to.equal(true) + done() + }) + }) + + it('should handle errors from the api', (done) => { + const peerKey = 'key of a peer not on the network' + const mockApi = nock('https://ipfs.io') + .post('/api/v0/dht/findpeer') + .query({ + 'arg': peerKey, + 'stream-channels': true + }) + .reply(502) + + nodeA.peerRouting.findPeer(peerKey, (err, peerInfo) => { + expect(err).to.exist() + expect(peerInfo).to.not.exist() + expect(mockApi.isDone()).to.equal(true) + done() + }) + }) + }) }) From 0ba0a64087e1073a001d89c8a243b511838b2380 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Fri, 17 Aug 2018 16:09:09 +0200 Subject: [PATCH 04/19] test: add jenkins module override --- ci/Jenkinsfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index a7da2e54f3..0131e073b7 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -1,2 +1,6 @@ // Warning: This file is automatically synced from https://github.com/ipfs/ci-sync so if you want to change it, please change it there and ask someone to sync all repositories. -javascript() +javascript([ + 'node_modules': [ + 'ipfs-api': 'github:ipfs/js-ipfs-api#test/easier-mocking' + ] +]) From e9900dbd646ae0dc05bb7b734519b32ac0711407 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Fri, 17 Aug 2018 16:37:26 +0200 Subject: [PATCH 05/19] fix: linting and bad test --- test/config.spec.js | 5 +++++ test/peer-routing.node.js | 7 +++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/test/config.spec.js b/test/config.spec.js index 60b8ca182f..3fbb9f2947 100644 --- a/test/config.spec.js +++ b/test/config.spec.js @@ -121,6 +121,11 @@ describe('configuration', () => { } } } + + expect(validateConfig(options).modules).to.deep.include({ + peerRouting: peerRouter, + contentRouting: contentRouter + }) }) it('should not allow for dht to be enabled without it being provided', () => { diff --git a/test/peer-routing.node.js b/test/peer-routing.node.js index a3230740bc..c59d06ad47 100644 --- a/test/peer-routing.node.js +++ b/test/peer-routing.node.js @@ -12,7 +12,6 @@ const DelegatedPeerRouter = require('libp2p-delegated-peer-routing') const sinon = require('sinon') const nock = require('nock') -const jsonPeerId = require('./fixtures/test-peer.json') const createNode = require('./utils/create-node') describe('.peerRouting', () => { @@ -143,7 +142,7 @@ describe('.peerRouting', () => { const mockApi = nock('https://ipfs.io') .post('/api/v0/dht/findpeer') .query({ - 'arg': peerKey, + arg: peerKey, 'stream-channels': true }) .reply(200, `{"Extra":"","ID":"some other id","Responses":null,"Type":0}\n{"Extra":"","ID":"","Responses":[{"Addrs":["/ip4/127.0.0.1/tcp/4001"],"ID":"${peerKey}"}],"Type":2}\n`, [ @@ -164,7 +163,7 @@ describe('.peerRouting', () => { const mockApi = nock('https://ipfs.io') .post('/api/v0/dht/findpeer') .query({ - 'arg': peerKey, + arg: peerKey, 'stream-channels': true }) .reply(200, `{"Extra":"","ID":"some other id","Responses":null,"Type":6}\n{"Extra":"","ID":"yet another id","Responses":null,"Type":0}\n{"Extra":"routing:not found","ID":"","Responses":null,"Type":3}\n`, [ @@ -185,7 +184,7 @@ describe('.peerRouting', () => { const mockApi = nock('https://ipfs.io') .post('/api/v0/dht/findpeer') .query({ - 'arg': peerKey, + arg: peerKey, 'stream-channels': true }) .reply(502) From df374a0656458eaa7e1d897a7eaf33150325b4ab Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Fri, 17 Aug 2018 16:38:35 +0200 Subject: [PATCH 06/19] test: module change --- ci/Jenkinsfile | 6 +- package.json | 3 + test/content-routing.node.js | 242 +++++++++++++++++++++++++++-------- test/peer-routing.node.js | 5 - test/utils/create-node.js | 11 +- 5 files changed, 203 insertions(+), 64 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 0131e073b7..a7da2e54f3 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -1,6 +1,2 @@ // Warning: This file is automatically synced from https://github.com/ipfs/ci-sync so if you want to change it, please change it there and ask someone to sync all repositories. -javascript([ - 'node_modules': [ - 'ipfs-api': 'github:ipfs/js-ipfs-api#test/easier-mocking' - ] -]) +javascript() diff --git a/package.json b/package.json index b11cff0622..79d1bd7e3b 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,9 @@ "webrtcsupport": "^2.2.0", "wrtc": "~0.2.0" }, + "resolutions": { + "ipfs-api": "github:ipfs/js-ipfs-api#test/easier-mocking" + }, "contributors": [ "Alan Shaw ", "Chris Bratlien ", diff --git a/test/content-routing.node.js b/test/content-routing.node.js index 3414a6174c..e930c0fd52 100644 --- a/test/content-routing.node.js +++ b/test/content-routing.node.js @@ -7,88 +7,226 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect const parallel = require('async/parallel') +const waterfall = require('async/waterfall') const _times = require('lodash.times') const CID = require('cids') +const DelegatedContentRouter = require('libp2p-delegated-content-routing') +const sinon = require('sinon') +const nock = require('nock') const createNode = require('./utils/create-node') +const createPeerInfo = createNode.createPeerInfo describe('.contentRouting', () => { - let nodeA - let nodeB - let nodeC - let nodeD - let nodeE - - before(function (done) { - this.timeout(5 * 1000) - const tasks = _times(5, () => (cb) => { - createNode('/ip4/0.0.0.0/tcp/0', { - config: { - EXPERIMENTAL: { - dht: true + describe('via the dht', () => { + let nodeA + let nodeB + let nodeC + let nodeD + let nodeE + + before(function (done) { + this.timeout(5 * 1000) + const tasks = _times(5, () => (cb) => { + createNode('/ip4/0.0.0.0/tcp/0', { + config: { + EXPERIMENTAL: { + dht: true + } } - } - }, (err, node) => { + }, (err, node) => { + expect(err).to.not.exist() + node.start((err) => cb(err, node)) + }) + }) + + parallel(tasks, (err, nodes) => { expect(err).to.not.exist() - node.start((err) => cb(err, node)) + nodeA = nodes[0] + nodeB = nodes[1] + nodeC = nodes[2] + nodeD = nodes[3] + nodeE = nodes[4] + + parallel([ + (cb) => nodeA.dial(nodeB.peerInfo, cb), + (cb) => nodeB.dial(nodeC.peerInfo, cb), + (cb) => nodeC.dial(nodeD.peerInfo, cb), + (cb) => nodeD.dial(nodeE.peerInfo, cb), + (cb) => nodeE.dial(nodeA.peerInfo, cb) + ], done) }) }) - parallel(tasks, (err, nodes) => { - expect(err).to.not.exist() - nodeA = nodes[0] - nodeB = nodes[1] - nodeC = nodes[2] - nodeD = nodes[3] - nodeE = nodes[4] - + after((done) => { parallel([ - (cb) => nodeA.dial(nodeB.peerInfo, cb), - (cb) => nodeB.dial(nodeC.peerInfo, cb), - (cb) => nodeC.dial(nodeD.peerInfo, cb), - (cb) => nodeD.dial(nodeE.peerInfo, cb), - (cb) => nodeE.dial(nodeA.peerInfo, cb) + (cb) => nodeA.stop(cb), + (cb) => nodeB.stop(cb), + (cb) => nodeC.stop(cb), + (cb) => nodeD.stop(cb), + (cb) => nodeE.stop(cb) ], done) }) - }) - after((done) => { - parallel([ - (cb) => nodeA.stop(cb), - (cb) => nodeB.stop(cb), - (cb) => nodeC.stop(cb), - (cb) => nodeD.stop(cb), - (cb) => nodeE.stop(cb) - ], done) + it('should use the nodes dht to provide', (done) => { + const stub = sinon.stub(nodeA._dht, 'provide').callsFake(() => { + stub.restore() + done() + }) + + nodeA.contentRouting.provide() + }) + + it('should use the nodes dht to find providers', (done) => { + const stub = sinon.stub(nodeA._dht, 'findProviders').callsFake(() => { + stub.restore() + done() + }) + + nodeA.contentRouting.findProviders() + }) + + describe('le ring', () => { + const cid = new CID('QmTp9VkYvnHyrqKQuFPiuZkiX9gPcqj6x5LJ1rmWuSySnL') + + it('let kbucket get filled', (done) => { + setTimeout(() => done(), 250) + }) + + it('nodeA.contentRouting.provide', (done) => { + nodeA.contentRouting.provide(cid, done) + }) + + it('nodeE.contentRouting.findProviders for existing record', (done) => { + nodeE.contentRouting.findProviders(cid, 5000, (err, providers) => { + expect(err).to.not.exist() + expect(providers).to.have.length.above(0) + done() + }) + }) + + it('nodeC.contentRouting.findProviders for non existing record (timeout)', (done) => { + const cid = new CID('QmTp9VkYvnHyrqKQuFPiuZkiX9gPcqj6x5LJ1rmWuSnnnn') + + nodeE.contentRouting.findProviders(cid, 5000, (err, providers) => { + expect(err).to.not.exist() + expect(providers).to.have.length(0) + done() + }) + }) + }) }) - describe('le ring', () => { - const cid = new CID('QmTp9VkYvnHyrqKQuFPiuZkiX9gPcqj6x5LJ1rmWuSySnL') + describe('via a delegate', () => { + let nodeA + let delegate - it('let kbucket get filled', (done) => { - setTimeout(() => done(), 250) + before((done) => { + waterfall([ + (cb) => { + createPeerInfo(cb) + }, + // Create the node using the delegate + (peerInfo, cb) => { + delegate = new DelegatedContentRouter(peerInfo.id, { + host: '0.0.0.0', + protocol: 'http', + port: '50082' + }) + createNode('/ip4/0.0.0.0/tcp/0', { + modules: { + contentRouting: delegate + } + }, (err, node) => { + expect(err).to.not.exist() + nodeA = node + nodeA.start(cb) + }) + } + ], done) }) - it('nodeA.contentRouting.provide', (done) => { - nodeA.contentRouting.provide(cid, done) + afterEach(() => { + nock.cleanAll() + nock.restore() }) - it('nodeE.contentRouting.findProviders for existing record', (done) => { - nodeE.contentRouting.findProviders(cid, 5000, (err, providers) => { - expect(err).to.not.exist() - expect(providers).to.have.length.above(0) + it('should use the delegate router to provide', (done) => { + const stub = sinon.stub(delegate, 'provide').callsFake(() => { + stub.restore() + done() + }) + nodeA.contentRouting.provide() + }) + + it('should use the delegate router to find providers', (done) => { + const stub = sinon.stub(delegate, 'findProviders').callsFake(() => { + stub.restore() done() }) + nodeA.contentRouting.findProviders() }) - it('nodeC.contentRouting.findProviders for non existing record (timeout)', (done) => { - const cid = new CID('QmTp9VkYvnHyrqKQuFPiuZkiX9gPcqj6x5LJ1rmWuSnnnn') + it('should be able to register as a provider', (done) => { + const cid = new CID('QmTp9VkYvnHyrqKQuFPiuZkiX9gPcqj6x5LJ1rmWuSySnL') + nock.recorder.rec() + // const mockApi = nock('https://ipfs.io') + // .post('/api/v0/dht/findpeer') + // .query({ + // arg: peerKey, + // 'stream-channels': true + // }) + // .reply(200, `{"Extra":"","ID":"some other id","Responses":null,"Type":0}\n{"Extra":"","ID":"","Responses":[{"Addrs":["/ip4/127.0.0.1/tcp/4001"],"ID":"${peerKey}"}],"Type":2}\n`, [ + // 'Content-Type', 'application/json', + // 'X-Chunked-Output', '1' + // ]) - nodeE.contentRouting.findProviders(cid, 5000, (err, providers) => { + nodeA.contentRouting.provide(cid, (err, data) => { expect(err).to.not.exist() - expect(providers).to.have.length(0) + expect(data).to.equal({}) + nock.restore() + // expect(mockApi.isDone()).to.equal(true) done() }) }) + + // it('should error when a peer cannot be found', (done) => { + // const peerKey = 'key of a peer not on the network' + // const mockApi = nock('https://ipfs.io') + // .post('/api/v0/dht/findpeer') + // .query({ + // arg: peerKey, + // 'stream-channels': true + // }) + // .reply(200, `{"Extra":"","ID":"some other id","Responses":null,"Type":6}\n{"Extra":"","ID":"yet another id","Responses":null,"Type":0}\n{"Extra":"routing:not found","ID":"","Responses":null,"Type":3}\n`, [ + // 'Content-Type', 'application/json', + // 'X-Chunked-Output', '1' + // ]) + + // nodeA.peerRouting.findPeer(peerKey, (err, peerInfo) => { + // expect(err).to.exist() + // expect(peerInfo).to.not.exist() + // expect(mockApi.isDone()).to.equal(true) + // done() + // }) + // }) + + // it('should handle errors from the api', (done) => { + // const peerKey = 'key of a peer not on the network' + // const mockApi = nock('https://ipfs.io') + // .post('/api/v0/dht/findpeer') + // .query({ + // arg: peerKey, + // 'stream-channels': true + // }) + // .reply(502) + + // nodeA.peerRouting.findPeer(peerKey, (err, peerInfo) => { + // expect(err).to.exist() + // expect(peerInfo).to.not.exist() + // expect(mockApi.isDone()).to.equal(true) + // done() + // }) + // }) }) }) diff --git a/test/peer-routing.node.js b/test/peer-routing.node.js index c59d06ad47..068139ec4b 100644 --- a/test/peer-routing.node.js +++ b/test/peer-routing.node.js @@ -112,11 +112,6 @@ describe('.peerRouting', () => { createNode('/ip4/0.0.0.0/tcp/0', { modules: { peerRouting: delegate - }, - config: { - EXPERIMENTAL: { - dht: true - } } }, (err, node) => { expect(err).to.not.exist() diff --git a/test/utils/create-node.js b/test/utils/create-node.js index 11564bfe85..3b9cf43bf2 100644 --- a/test/utils/create-node.js +++ b/test/utils/create-node.js @@ -21,8 +21,7 @@ function createNode (multiaddrs, options, callback) { } waterfall([ - (cb) => PeerId.create({ bits: 512 }, cb), - (peerId, cb) => PeerInfo.create(peerId, cb), + (cb) => createPeerInfo(cb), (peerInfo, cb) => { multiaddrs.map((ma) => peerInfo.multiaddrs.add(ma)) options.peerInfo = peerInfo @@ -31,4 +30,12 @@ function createNode (multiaddrs, options, callback) { ], callback) } +function createPeerInfo (callback) { + waterfall([ + (cb) => PeerId.create({ bits: 512 }, cb), + (peerId, cb) => PeerInfo.create(peerId, cb) + ], callback) +} + module.exports = createNode +module.exports.createPeerInfo = createPeerInfo From 9285e269754b2cd4347bac5d729daa192521a93e Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Tue, 21 Aug 2018 17:51:39 +0200 Subject: [PATCH 07/19] test: add tests for content routing --- test/content-routing.node.js | 134 ++++++++++++++++++----------------- 1 file changed, 68 insertions(+), 66 deletions(-) diff --git a/test/content-routing.node.js b/test/content-routing.node.js index e930c0fd52..62ed01a5e4 100644 --- a/test/content-routing.node.js +++ b/test/content-routing.node.js @@ -13,6 +13,8 @@ const CID = require('cids') const DelegatedContentRouter = require('libp2p-delegated-content-routing') const sinon = require('sinon') const nock = require('nock') +const ma = require('multiaddr') +const Node = require('./utils/bundle-nodejs') const createNode = require('./utils/create-node') const createPeerInfo = createNode.createPeerInfo @@ -131,25 +133,31 @@ describe('.contentRouting', () => { delegate = new DelegatedContentRouter(peerInfo.id, { host: '0.0.0.0', protocol: 'http', - port: '50082' - }) - createNode('/ip4/0.0.0.0/tcp/0', { + port: 60197 + }, [ + ma('/ip4/0.0.0.0/tcp/60194') + ]) + nodeA = new Node({ + peerInfo, modules: { contentRouting: delegate + }, + config: { + relay: { + enabled: true, + hop: { + enabled: true, + active: false + } + } } - }, (err, node) => { - expect(err).to.not.exist() - nodeA = node - nodeA.start(cb) }) + nodeA.start(cb) } ], done) }) - afterEach(() => { - nock.cleanAll() - nock.restore() - }) + afterEach(() => nock.cleanAll) it('should use the delegate router to provide', (done) => { const stub = sinon.stub(delegate, 'provide').callsFake(() => { @@ -168,65 +176,59 @@ describe('.contentRouting', () => { }) it('should be able to register as a provider', (done) => { - const cid = new CID('QmTp9VkYvnHyrqKQuFPiuZkiX9gPcqj6x5LJ1rmWuSySnL') - nock.recorder.rec() - // const mockApi = nock('https://ipfs.io') - // .post('/api/v0/dht/findpeer') - // .query({ - // arg: peerKey, - // 'stream-channels': true - // }) - // .reply(200, `{"Extra":"","ID":"some other id","Responses":null,"Type":0}\n{"Extra":"","ID":"","Responses":[{"Addrs":["/ip4/127.0.0.1/tcp/4001"],"ID":"${peerKey}"}],"Type":2}\n`, [ - // 'Content-Type', 'application/json', - // 'X-Chunked-Output', '1' - // ]) - - nodeA.contentRouting.provide(cid, (err, data) => { + const cid = new CID('QmU621oD8AhHw6t25vVyfYKmL9VV3PTgc52FngEhTGACFB') + const mockApi = nock('http://0.0.0.0:60197') + // mock the swarm connect + .post('/api/v0/swarm/connect') + .query({ + arg: `/ip4/0.0.0.0/tcp/60194/p2p-circuit/ipfs/${nodeA.peerInfo.id.toB58String()}`, + 'stream-channels': true + }) + .reply(200, { + Strings: [`connect ${nodeA.peerInfo.id.toB58String()} success`] + }, ['Content-Type', 'application/json']) + // mock the refs call + .post('/api/v0/refs') + .query({ + recursive: true, + arg: cid.toBaseEncodedString(), + 'stream-channels': true + }) + .reply(200, null, [ + 'Content-Type', 'application/json', + 'X-Chunked-Output', '1' + ]) + + nodeA.contentRouting.provide(cid, (err) => { expect(err).to.not.exist() - expect(data).to.equal({}) - nock.restore() - // expect(mockApi.isDone()).to.equal(true) + expect(mockApi.isDone()).to.equal(true) done() }) }) - // it('should error when a peer cannot be found', (done) => { - // const peerKey = 'key of a peer not on the network' - // const mockApi = nock('https://ipfs.io') - // .post('/api/v0/dht/findpeer') - // .query({ - // arg: peerKey, - // 'stream-channels': true - // }) - // .reply(200, `{"Extra":"","ID":"some other id","Responses":null,"Type":6}\n{"Extra":"","ID":"yet another id","Responses":null,"Type":0}\n{"Extra":"routing:not found","ID":"","Responses":null,"Type":3}\n`, [ - // 'Content-Type', 'application/json', - // 'X-Chunked-Output', '1' - // ]) - - // nodeA.peerRouting.findPeer(peerKey, (err, peerInfo) => { - // expect(err).to.exist() - // expect(peerInfo).to.not.exist() - // expect(mockApi.isDone()).to.equal(true) - // done() - // }) - // }) - - // it('should handle errors from the api', (done) => { - // const peerKey = 'key of a peer not on the network' - // const mockApi = nock('https://ipfs.io') - // .post('/api/v0/dht/findpeer') - // .query({ - // arg: peerKey, - // 'stream-channels': true - // }) - // .reply(502) - - // nodeA.peerRouting.findPeer(peerKey, (err, peerInfo) => { - // expect(err).to.exist() - // expect(peerInfo).to.not.exist() - // expect(mockApi.isDone()).to.equal(true) - // done() - // }) - // }) + it('should be able to find providers', (done) => { + const cid = new CID('QmU621oD8AhHw6t25vVyfYKmL9VV3PTgc52FngEhTGACFB') + const provider = 'QmZNgCqZCvTsi3B4Vt7gsSqpkqDpE7M2Y9TDmEhbDb4ceF' + const mockApi = nock('http://0.0.0.0:60197') + .post('/api/v0/dht/findprovs') + .query({ + arg: cid.toBaseEncodedString(), + 'stream-channels': true + }) + .reply(200, `{"Extra":"","ID":"QmWKqWXCtRXEeCQTo3FoZ7g4AfnGiauYYiczvNxFCHicbB","Responses":[{"Addrs":["/ip4/0.0.0.0/tcp/0"],"ID":"${provider}"}],"Type":1}\n`, [ + 'Content-Type', 'application/json', + 'X-Chunked-Output', '1' + ]) + + nodeA.contentRouting.findProviders(cid.toBaseEncodedString(), (err, response) => { + expect(err).to.not.exist() + expect(response).to.have.length(1) + expect(response[0]).to.include({ + id: provider + }) + expect(mockApi.isDone()).to.equal(true) + done() + }) + }) }) }) From 86ab1a02cd5c632e11317cc7356671c335d19358 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Tue, 21 Aug 2018 19:14:52 +0200 Subject: [PATCH 08/19] test: add error case tests --- package.json | 2 +- test/content-routing.node.js | 157 ++++++++++++++++++++++------------- 2 files changed, 100 insertions(+), 59 deletions(-) diff --git a/package.json b/package.json index 79d1bd7e3b..94463704fa 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ "wrtc": "~0.2.0" }, "resolutions": { - "ipfs-api": "github:ipfs/js-ipfs-api#test/easier-mocking" + "ipfs-api": "github:ipfs/js-ipfs-api#master" }, "contributors": [ "Alan Shaw ", diff --git a/test/content-routing.node.js b/test/content-routing.node.js index 62ed01a5e4..b0146c2497 100644 --- a/test/content-routing.node.js +++ b/test/content-routing.node.js @@ -159,75 +159,116 @@ describe('.contentRouting', () => { afterEach(() => nock.cleanAll) - it('should use the delegate router to provide', (done) => { - const stub = sinon.stub(delegate, 'provide').callsFake(() => { - stub.restore() - done() + describe('provide', () => { + it('should use the delegate router to provide', (done) => { + const stub = sinon.stub(delegate, 'provide').callsFake(() => { + stub.restore() + done() + }) + nodeA.contentRouting.provide() }) - nodeA.contentRouting.provide() - }) - it('should use the delegate router to find providers', (done) => { - const stub = sinon.stub(delegate, 'findProviders').callsFake(() => { - stub.restore() - done() - }) - nodeA.contentRouting.findProviders() - }) + it('should be able to register as a provider', (done) => { + const cid = new CID('QmU621oD8AhHw6t25vVyfYKmL9VV3PTgc52FngEhTGACFB') + const mockApi = nock('http://0.0.0.0:60197') + // mock the swarm connect + .post('/api/v0/swarm/connect') + .query({ + arg: `/ip4/0.0.0.0/tcp/60194/p2p-circuit/ipfs/${nodeA.peerInfo.id.toB58String()}`, + 'stream-channels': true + }) + .reply(200, { + Strings: [`connect ${nodeA.peerInfo.id.toB58String()} success`] + }, ['Content-Type', 'application/json']) + // mock the refs call + .post('/api/v0/refs') + .query({ + recursive: true, + arg: cid.toBaseEncodedString(), + 'stream-channels': true + }) + .reply(200, null, [ + 'Content-Type', 'application/json', + 'X-Chunked-Output', '1' + ]) - it('should be able to register as a provider', (done) => { - const cid = new CID('QmU621oD8AhHw6t25vVyfYKmL9VV3PTgc52FngEhTGACFB') - const mockApi = nock('http://0.0.0.0:60197') - // mock the swarm connect - .post('/api/v0/swarm/connect') - .query({ - arg: `/ip4/0.0.0.0/tcp/60194/p2p-circuit/ipfs/${nodeA.peerInfo.id.toB58String()}`, - 'stream-channels': true - }) - .reply(200, { - Strings: [`connect ${nodeA.peerInfo.id.toB58String()} success`] - }, ['Content-Type', 'application/json']) - // mock the refs call - .post('/api/v0/refs') - .query({ - recursive: true, - arg: cid.toBaseEncodedString(), - 'stream-channels': true + nodeA.contentRouting.provide(cid, (err) => { + expect(err).to.not.exist() + expect(mockApi.isDone()).to.equal(true) + done() }) - .reply(200, null, [ - 'Content-Type', 'application/json', - 'X-Chunked-Output', '1' - ]) + }) - nodeA.contentRouting.provide(cid, (err) => { - expect(err).to.not.exist() - expect(mockApi.isDone()).to.equal(true) - done() + it('should handle errors when registering as a provider', (done) => { + const cid = new CID('QmU621oD8AhHw6t25vVyfYKmL9VV3PTgc52FngEhTGACFB') + const mockApi = nock('http://0.0.0.0:60197') + // mock the swarm connect + .post('/api/v0/swarm/connect') + .query({ + arg: `/ip4/0.0.0.0/tcp/60194/p2p-circuit/ipfs/${nodeA.peerInfo.id.toB58String()}`, + 'stream-channels': true + }) + .reply(502, 'Bad Gateway', ['Content-Type', 'application/json']) + + nodeA.contentRouting.provide(cid, (err) => { + expect(err).to.exist() + expect(mockApi.isDone()).to.equal(true) + done() + }) }) }) - it('should be able to find providers', (done) => { - const cid = new CID('QmU621oD8AhHw6t25vVyfYKmL9VV3PTgc52FngEhTGACFB') - const provider = 'QmZNgCqZCvTsi3B4Vt7gsSqpkqDpE7M2Y9TDmEhbDb4ceF' - const mockApi = nock('http://0.0.0.0:60197') - .post('/api/v0/dht/findprovs') - .query({ - arg: cid.toBaseEncodedString(), - 'stream-channels': true + describe('find providers', () => { + it('should use the delegate router to find providers', (done) => { + const stub = sinon.stub(delegate, 'findProviders').callsFake(() => { + stub.restore() + done() }) - .reply(200, `{"Extra":"","ID":"QmWKqWXCtRXEeCQTo3FoZ7g4AfnGiauYYiczvNxFCHicbB","Responses":[{"Addrs":["/ip4/0.0.0.0/tcp/0"],"ID":"${provider}"}],"Type":1}\n`, [ - 'Content-Type', 'application/json', - 'X-Chunked-Output', '1' - ]) + nodeA.contentRouting.findProviders() + }) - nodeA.contentRouting.findProviders(cid.toBaseEncodedString(), (err, response) => { - expect(err).to.not.exist() - expect(response).to.have.length(1) - expect(response[0]).to.include({ - id: provider + it('should be able to find providers', (done) => { + const cid = new CID('QmU621oD8AhHw6t25vVyfYKmL9VV3PTgc52FngEhTGACFB') + const provider = 'QmZNgCqZCvTsi3B4Vt7gsSqpkqDpE7M2Y9TDmEhbDb4ceF' + const mockApi = nock('http://0.0.0.0:60197') + .post('/api/v0/dht/findprovs') + .query({ + arg: cid.toBaseEncodedString(), + 'stream-channels': true + }) + .reply(200, `{"Extra":"","ID":"QmWKqWXCtRXEeCQTo3FoZ7g4AfnGiauYYiczvNxFCHicbB","Responses":[{"Addrs":["/ip4/0.0.0.0/tcp/0"],"ID":"${provider}"}],"Type":1}\n`, [ + 'Content-Type', 'application/json', + 'X-Chunked-Output', '1' + ]) + + nodeA.contentRouting.findProviders(cid, (err, response) => { + expect(err).to.not.exist() + expect(response).to.have.length(1) + expect(response[0]).to.include({ + id: provider + }) + expect(mockApi.isDone()).to.equal(true) + done() + }) + }) + + it('should handle errors when finding providers', (done) => { + const cid = new CID('QmU621oD8AhHw6t25vVyfYKmL9VV3PTgc52FngEhTGACFB') + const mockApi = nock('http://0.0.0.0:60197') + .post('/api/v0/dht/findprovs') + .query({ + arg: cid.toBaseEncodedString(), + 'stream-channels': true + }) + .reply(502, 'Bad Gateway', [ + 'X-Chunked-Output', '1' + ]) + + nodeA.contentRouting.findProviders(cid, (err) => { + expect(err).to.exist() + expect(mockApi.isDone()).to.equal(true) + done() }) - expect(mockApi.isDone()).to.equal(true) - done() }) }) }) From 59bc0a0f749259ed9195872f6f2eb8de936038cb Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Thu, 23 Aug 2018 13:47:07 +0200 Subject: [PATCH 09/19] feat: support multiple peer and content routing modules --- src/config.js | 4 +- src/content-routing.js | 63 ++++++++++++++++++++--- src/index.js | 6 +-- src/peer-routing.js | 43 ++++++++++++++-- test/config.spec.js | 8 +-- test/content-routing.node.js | 99 +++++++++++++++++++++++++++++++++++- test/peer-routing.node.js | 75 ++++++++++++++++++++++++++- 7 files changed, 276 insertions(+), 22 deletions(-) diff --git a/src/config.js b/src/config.js index 2b7485254a..ac95d1a7d2 100644 --- a/src/config.js +++ b/src/config.js @@ -14,10 +14,10 @@ const OptionsSchema = Joi.object({ connProtector: Joi.object().keys({ protect: Joi.func().required() }).unknown(), - contentRouting: Joi.object(), + contentRouting: Joi.array().items(Joi.object()).allow(null), dht: ModuleSchema.allow(null), peerDiscovery: Joi.array().items(ModuleSchema).allow(null), - peerRouting: Joi.object(), + peerRouting: Joi.array().items(Joi.object()).allow(null), streamMuxer: Joi.array().items(ModuleSchema).allow(null), transport: Joi.array().items(ModuleSchema).min(1).required() }).required(), diff --git a/src/content-routing.js b/src/content-routing.js index 559541ed45..a30e31bb50 100644 --- a/src/content-routing.js +++ b/src/content-routing.js @@ -1,20 +1,71 @@ 'use strict' +const tryEach = require('async/tryEach') +const parallel = require('async/parallel') + module.exports = (node) => { + const routers = node._modules.contentRouting || [] + + // If we have the dht, make it first + if (node._dht) { + routers.unshift(node._dht) + } + return { + /** + * Iterates over all content routers in series to find providers of the given key. + * Once a content router succeeds, iteration will stop. + * + * @param {CID} key The CID key of the content to find + * @param {number} timeout How long the query should run + * @param {function(Error, Result)} callback + * @returns {void} + */ findProviders: (key, timeout, callback) => { - if (!node._dht) { - return callback(new Error('DHT is not available')) + if (routers.length === 0) { + return callback(new Error('No content routers available')) } - node._dht.findProviders(key, timeout, callback) + const tasks = routers.map((router) => { + return (cb) => router.findProviders(key, timeout, (err, results) => { + if (err) { + return cb(err) + } + + // If we don't have any results, we need to provide an error to keep trying + if (!results || Object.keys(results).length === 0) { + return cb(true, null) + } + + cb(null, results) + }) + }) + + tryEach(tasks, (err, results) => { + if (err && err !== true) { + return callback(err) + } + results = results || [] + callback(null, results) + }) }, + + /** + * Iterates over all content routers in parallel to notify it is + * a provider of the given key. + * + * @param {CID} key The CID key of the content to find + * @param {function(Error)} callback + * @returns {void} + */ provide: (key, callback) => { - if (!node._dht) { - return callback(new Error('DHT is not available')) + if (routers.length === 0) { + return callback(new Error('No content routers available')) } - node._dht.provide(key, callback) + parallel(routers.map((router) => { + return (cb) => router.provide(key, cb) + }), callback) } } } diff --git a/src/index.js b/src/index.js index 91a2c74309..5372f9966c 100644 --- a/src/index.js +++ b/src/index.js @@ -102,9 +102,9 @@ class Node extends EventEmitter { } // Attach remaining APIs - // If peer or content routing modules have been provided, use those, otherwise use the dht - this.peerRouting = this._modules.peerRouting || peerRouting(this) - this.contentRouting = this._modules.contentRouting || contentRouting(this) + // peer and content routing will automatically get modules from _modules and _dht + this.peerRouting = peerRouting(this) + this.contentRouting = contentRouting(this) this.dht = dht(this) this._getPeerInfo = getPeerInfo(this) diff --git a/src/peer-routing.js b/src/peer-routing.js index 3a48d075ee..183f9c15ec 100644 --- a/src/peer-routing.js +++ b/src/peer-routing.js @@ -1,13 +1,50 @@ 'use strict' +const tryEach = require('async/tryEach') + module.exports = (node) => { + const routers = node._modules.peerRouting || [] + + // If we have the dht, make it first + if (node._dht) { + routers.unshift(node._dht) + } + return { + /** + * Iterates over all peer routers in series to find the given peer. + * + * @param {String} id The id of the peer to find + * @param {function(Error, Result)} + * @returns {void} + */ findPeer: (id, callback) => { - if (!node._dht) { - return callback(new Error('DHT is not available')) + if (routers.length === 0) { + return callback(new Error('No peer routers available')) } - node._dht.findPeer(id, callback) + const tasks = routers.map((router) => { + return (cb) => router.findPeer(id, (err, result) => { + if (err) { + return cb(err) + } + + // If we don't have a result, we need to provide an error to keep trying + if (!result || Object.keys(result).length === 0) { + return cb(true, null) + } + + cb(null, result) + }) + }) + + tryEach(tasks, (err, results) => { + if (err && err !== true) { + return callback(err) + } + results = results || null + callback(null, results) + }) } } } diff --git a/test/config.spec.js b/test/config.spec.js index 3fbb9f2947..3b4c687dcd 100644 --- a/test/config.spec.js +++ b/test/config.spec.js @@ -109,8 +109,8 @@ describe('configuration', () => { modules: { transport: [ WS ], peerDiscovery: [ Bootstrap ], - peerRouting: peerRouter, - contentRouting: contentRouter + peerRouting: [ peerRouter ], + contentRouting: [ contentRouter ] }, config: { peerDiscovery: { @@ -123,8 +123,8 @@ describe('configuration', () => { } expect(validateConfig(options).modules).to.deep.include({ - peerRouting: peerRouter, - contentRouting: contentRouter + peerRouting: [ peerRouter ], + contentRouting: [ contentRouter ] }) }) diff --git a/test/content-routing.node.js b/test/content-routing.node.js index b0146c2497..9ca1243208 100644 --- a/test/content-routing.node.js +++ b/test/content-routing.node.js @@ -140,7 +140,7 @@ describe('.contentRouting', () => { nodeA = new Node({ peerInfo, modules: { - contentRouting: delegate + contentRouting: [ delegate ] }, config: { relay: { @@ -157,7 +157,8 @@ describe('.contentRouting', () => { ], done) }) - afterEach(() => nock.cleanAll) + after((done) => nodeA.stop(done)) + afterEach(() => nock.cleanAll()) describe('provide', () => { it('should use the delegate router to provide', (done) => { @@ -272,4 +273,98 @@ describe('.contentRouting', () => { }) }) }) + + describe('via the dht and a delegate', () => { + let nodeA + let delegate + + before((done) => { + waterfall([ + (cb) => { + createPeerInfo(cb) + }, + // Create the node using the delegate + (peerInfo, cb) => { + delegate = new DelegatedContentRouter(peerInfo.id, { + host: '0.0.0.0', + protocol: 'http', + port: 60197 + }, [ + ma('/ip4/0.0.0.0/tcp/60194') + ]) + nodeA = new Node({ + peerInfo, + modules: { + contentRouting: [ delegate ] + }, + config: { + relay: { + enabled: true, + hop: { + enabled: true, + active: false + } + }, + EXPERIMENTAL: { + dht: true + } + } + }) + nodeA.start(cb) + } + ], done) + }) + + after((done) => nodeA.stop(done)) + + describe('provide', () => { + it('should use both the dht and delegate router to provide', (done) => { + const dhtStub = sinon.stub(nodeA._dht, 'provide').callsFake(() => {}) + const delegateStub = sinon.stub(delegate, 'provide').callsFake(() => { + expect(dhtStub.calledOnce).to.equal(true) + expect(delegateStub.calledOnce).to.equal(true) + delegateStub.restore() + dhtStub.restore() + done() + }) + nodeA.contentRouting.provide() + }) + }) + + describe('findProviders', () => { + it('should only use the dht if it finds providers', (done) => { + const results = [true] + const dhtStub = sinon.stub(nodeA._dht, 'findProviders').callsArgWith(2, null, results) + const delegateStub = sinon.stub(delegate, 'findProviders').throws(() => { + return new Error('the delegate should not have been called') + }) + + nodeA.contentRouting.findProviders('a cid', 5000, (err, results) => { + expect(err).to.not.exist() + expect(results).to.equal(results) + expect(dhtStub.calledOnce).to.equal(true) + expect(delegateStub.notCalled).to.equal(true) + delegateStub.restore() + dhtStub.restore() + done() + }) + }) + + it('should use the delegate if the dht fails to find providers', (done) => { + const results = [true] + const dhtStub = sinon.stub(nodeA._dht, 'findProviders').callsArgWith(2, null, []) + const delegateStub = sinon.stub(delegate, 'findProviders').callsArgWith(2, null, results) + + nodeA.contentRouting.findProviders('a cid', 5000, (err, results) => { + expect(err).to.not.exist() + expect(results).to.deep.equal(results) + expect(dhtStub.calledOnce).to.equal(true) + expect(delegateStub.calledOnce).to.equal(true) + delegateStub.restore() + dhtStub.restore() + done() + }) + }) + }) + }) }) diff --git a/test/peer-routing.node.js b/test/peer-routing.node.js index 068139ec4b..4d8499afa1 100644 --- a/test/peer-routing.node.js +++ b/test/peer-routing.node.js @@ -111,7 +111,7 @@ describe('.peerRouting', () => { }) createNode('/ip4/0.0.0.0/tcp/0', { modules: { - peerRouting: delegate + peerRouting: [ delegate ] } }, (err, node) => { expect(err).to.not.exist() @@ -122,7 +122,8 @@ describe('.peerRouting', () => { ], done) }) - afterEach(nock.cleanAll) + after((done) => nodeA.stop(done)) + afterEach(() => nock.cleanAll()) it('should use the delegate router to find peers', (done) => { const stub = sinon.stub(delegate, 'findPeer').callsFake(() => { @@ -192,4 +193,74 @@ describe('.peerRouting', () => { }) }) }) + + describe('via the dht and a delegate', () => { + let nodeA + let delegate + + before((done) => { + parallel([ + // Create the node using the delegate + (cb) => { + delegate = new DelegatedPeerRouter({ + host: 'ipfs.io', + protocol: 'https', + port: '443' + }) + createNode('/ip4/0.0.0.0/tcp/0', { + modules: { + peerRouting: [ delegate ] + }, + config: { + EXPERIMENTAL: { + dht: true + } + } + }, (err, node) => { + expect(err).to.not.exist() + nodeA = node + nodeA.start(cb) + }) + } + ], done) + }) + + after((done) => nodeA.stop(done)) + + describe('findPeer', () => { + it('should only use the dht if it find the peer', (done) => { + const results = [true] + const dhtStub = sinon.stub(nodeA._dht, 'findPeer').callsArgWith(1, null, results) + const delegateStub = sinon.stub(delegate, 'findPeer').throws(() => { + return new Error('the delegate should not have been called') + }) + + nodeA.peerRouting.findPeer('a peer id', (err, results) => { + expect(err).to.not.exist() + expect(results).to.equal(results) + expect(dhtStub.calledOnce).to.equal(true) + expect(delegateStub.notCalled).to.equal(true) + delegateStub.restore() + dhtStub.restore() + done() + }) + }) + + it('should use the delegate if the dht fails to find the peer', (done) => { + const results = [true] + const dhtStub = sinon.stub(nodeA._dht, 'findPeer').callsArgWith(1, null, undefined) + const delegateStub = sinon.stub(delegate, 'findPeer').callsArgWith(1, null, results) + + nodeA.peerRouting.findPeer('a peer id', (err, results) => { + expect(err).to.not.exist() + expect(results).to.deep.equal(results) + expect(dhtStub.calledOnce).to.equal(true) + expect(delegateStub.calledOnce).to.equal(true) + delegateStub.restore() + dhtStub.restore() + done() + }) + }) + }) + }) }) From fb4054a9a5ffb06fbbbba12cb4dcac9d21391785 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Thu, 23 Aug 2018 15:22:42 +0200 Subject: [PATCH 10/19] fix: linting --- src/content-routing.js | 6 ++++-- src/peer-routing.js | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/content-routing.js b/src/content-routing.js index a30e31bb50..5a18dbffbd 100644 --- a/src/content-routing.js +++ b/src/content-routing.js @@ -34,7 +34,9 @@ module.exports = (node) => { // If we don't have any results, we need to provide an error to keep trying if (!results || Object.keys(results).length === 0) { - return cb(true, null) + return cb(Object.assign(new Error('not found'), { + code: 'NOT_FOUND' + }), null) } cb(null, results) @@ -42,7 +44,7 @@ module.exports = (node) => { }) tryEach(tasks, (err, results) => { - if (err && err !== true) { + if (err && err.code !== 'NOT_FOUND') { return callback(err) } results = results || [] diff --git a/src/peer-routing.js b/src/peer-routing.js index 183f9c15ec..e4f60f52b8 100644 --- a/src/peer-routing.js +++ b/src/peer-routing.js @@ -15,7 +15,7 @@ module.exports = (node) => { * Iterates over all peer routers in series to find the given peer. * * @param {String} id The id of the peer to find - * @param {function(Error, Result)} + * @param {function(Error, Result)} callback * @returns {void} */ findPeer: (id, callback) => { @@ -31,7 +31,9 @@ module.exports = (node) => { // If we don't have a result, we need to provide an error to keep trying if (!result || Object.keys(result).length === 0) { - return cb(true, null) + return cb(Object.assign(new Error('not found'), { + code: 'NOT_FOUND' + }), null) } cb(null, result) @@ -39,7 +41,7 @@ module.exports = (node) => { }) tryEach(tasks, (err, results) => { - if (err && err !== true) { + if (err && err.code !== 'NOT_FOUND') { return callback(err) } results = results || null From ef9961aaee02d6c145f76f31b5a2c0afaebc3d2c Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 27 Aug 2018 16:42:56 +0200 Subject: [PATCH 11/19] chore: update routing modules and test params --- test/content-routing.node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/content-routing.node.js b/test/content-routing.node.js index 9ca1243208..f9db54d7bb 100644 --- a/test/content-routing.node.js +++ b/test/content-routing.node.js @@ -242,7 +242,7 @@ describe('.contentRouting', () => { 'X-Chunked-Output', '1' ]) - nodeA.contentRouting.findProviders(cid, (err, response) => { + nodeA.contentRouting.findProviders(cid, 1000, (err, response) => { expect(err).to.not.exist() expect(response).to.have.length(1) expect(response[0]).to.include({ From 194511d5abab4cbe1770d03655dbaa9a673c85d4 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Tue, 28 Aug 2018 13:48:02 +0200 Subject: [PATCH 12/19] chore: update deps --- package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/package.json b/package.json index 94463704fa..b11cff0622 100644 --- a/package.json +++ b/package.json @@ -80,9 +80,6 @@ "webrtcsupport": "^2.2.0", "wrtc": "~0.2.0" }, - "resolutions": { - "ipfs-api": "github:ipfs/js-ipfs-api#master" - }, "contributors": [ "Alan Shaw ", "Chris Bratlien ", From f56b888790cce0a4de9b80bc2b5d0295a9e6ba28 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 24 Sep 2018 15:02:23 +0200 Subject: [PATCH 13/19] feat: add timeout support to peer and content routing --- package.json | 4 ++-- src/peer-routing.js | 10 ++++++++-- test/content-routing.node.js | 6 +++--- test/peer-routing.node.js | 13 ++++++++----- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index b11cff0622..dde1ab52f0 100644 --- a/package.json +++ b/package.json @@ -60,8 +60,8 @@ "dirty-chai": "^2.0.1", "electron-webrtc": "~0.3.0", "libp2p-circuit": "~0.2.1", - "libp2p-delegated-content-routing": "github:libp2p/js-libp2p-delegated-content-routing#master", - "libp2p-delegated-peer-routing": "github:libp2p/js-libp2p-delegated-peer-routing#master", + "libp2p-delegated-content-routing": "github:libp2p/js-libp2p-delegated-content-routing#feat/timeout-defaults", + "libp2p-delegated-peer-routing": "github:libp2p/js-libp2p-delegated-peer-routing#feat/timeout-defaults", "libp2p-kad-dht": "~0.10.5", "libp2p-mdns": "~0.12.0", "libp2p-mplex": "~0.8.2", diff --git a/src/peer-routing.js b/src/peer-routing.js index e4f60f52b8..066cf31b3d 100644 --- a/src/peer-routing.js +++ b/src/peer-routing.js @@ -15,16 +15,22 @@ module.exports = (node) => { * Iterates over all peer routers in series to find the given peer. * * @param {String} id The id of the peer to find + * @param {number} timeout How long the query should run * @param {function(Error, Result)} callback * @returns {void} */ - findPeer: (id, callback) => { + findPeer: (id, timeout, callback) => { if (routers.length === 0) { return callback(new Error('No peer routers available')) } + if (typeof timeout === 'function') { + callback = timeout + timeout = null + } + const tasks = routers.map((router) => { - return (cb) => router.findPeer(id, (err, result) => { + return (cb) => router.findPeer(id, timeout, (err, result) => { if (err) { return cb(err) } diff --git a/test/content-routing.node.js b/test/content-routing.node.js index f9db54d7bb..37eddc27c2 100644 --- a/test/content-routing.node.js +++ b/test/content-routing.node.js @@ -235,6 +235,7 @@ describe('.contentRouting', () => { .post('/api/v0/dht/findprovs') .query({ arg: cid.toBaseEncodedString(), + timeout: '1000ms', 'stream-channels': true }) .reply(200, `{"Extra":"","ID":"QmWKqWXCtRXEeCQTo3FoZ7g4AfnGiauYYiczvNxFCHicbB","Responses":[{"Addrs":["/ip4/0.0.0.0/tcp/0"],"ID":"${provider}"}],"Type":1}\n`, [ @@ -245,9 +246,7 @@ describe('.contentRouting', () => { nodeA.contentRouting.findProviders(cid, 1000, (err, response) => { expect(err).to.not.exist() expect(response).to.have.length(1) - expect(response[0]).to.include({ - id: provider - }) + expect(response[0].id.toB58String()).to.equal(provider) expect(mockApi.isDone()).to.equal(true) done() }) @@ -259,6 +258,7 @@ describe('.contentRouting', () => { .post('/api/v0/dht/findprovs') .query({ arg: cid.toBaseEncodedString(), + timeout: '30000ms', 'stream-channels': true }) .reply(502, 'Bad Gateway', [ diff --git a/test/peer-routing.node.js b/test/peer-routing.node.js index 4d8499afa1..7a4cc167e9 100644 --- a/test/peer-routing.node.js +++ b/test/peer-routing.node.js @@ -134,11 +134,12 @@ describe('.peerRouting', () => { }) it('should be able to find a peer', (done) => { - const peerKey = 'key of a peer on the network' + const peerKey = 'QmTp9VkYvnHyrqKQuFPiuZkiX9gPcqj6x5LJ1rmWuSySnL' const mockApi = nock('https://ipfs.io') .post('/api/v0/dht/findpeer') .query({ arg: peerKey, + timeout: '30000ms', 'stream-channels': true }) .reply(200, `{"Extra":"","ID":"some other id","Responses":null,"Type":0}\n{"Extra":"","ID":"","Responses":[{"Addrs":["/ip4/127.0.0.1/tcp/4001"],"ID":"${peerKey}"}],"Type":2}\n`, [ @@ -148,7 +149,7 @@ describe('.peerRouting', () => { nodeA.peerRouting.findPeer(peerKey, (err, peerInfo) => { expect(err).to.not.exist() - expect(peerInfo.id).to.equal(peerKey) + expect(peerInfo.id.toB58String()).to.equal(peerKey) expect(mockApi.isDone()).to.equal(true) done() }) @@ -160,6 +161,7 @@ describe('.peerRouting', () => { .post('/api/v0/dht/findpeer') .query({ arg: peerKey, + timeout: '30000ms', 'stream-channels': true }) .reply(200, `{"Extra":"","ID":"some other id","Responses":null,"Type":6}\n{"Extra":"","ID":"yet another id","Responses":null,"Type":0}\n{"Extra":"routing:not found","ID":"","Responses":null,"Type":3}\n`, [ @@ -181,6 +183,7 @@ describe('.peerRouting', () => { .post('/api/v0/dht/findpeer') .query({ arg: peerKey, + timeout: '30000ms', 'stream-channels': true }) .reply(502) @@ -230,7 +233,7 @@ describe('.peerRouting', () => { describe('findPeer', () => { it('should only use the dht if it find the peer', (done) => { const results = [true] - const dhtStub = sinon.stub(nodeA._dht, 'findPeer').callsArgWith(1, null, results) + const dhtStub = sinon.stub(nodeA._dht, 'findPeer').callsArgWith(2, null, results) const delegateStub = sinon.stub(delegate, 'findPeer').throws(() => { return new Error('the delegate should not have been called') }) @@ -248,8 +251,8 @@ describe('.peerRouting', () => { it('should use the delegate if the dht fails to find the peer', (done) => { const results = [true] - const dhtStub = sinon.stub(nodeA._dht, 'findPeer').callsArgWith(1, null, undefined) - const delegateStub = sinon.stub(delegate, 'findPeer').callsArgWith(1, null, results) + const dhtStub = sinon.stub(nodeA._dht, 'findPeer').callsArgWith(2, null, undefined) + const delegateStub = sinon.stub(delegate, 'findPeer').callsArgWith(2, null, results) nodeA.peerRouting.findPeer('a peer id', (err, results) => { expect(err).to.not.exist() From 8d5f410a9cbb4c41ce087c9e7eede94cb4dbe7ea Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Mon, 24 Sep 2018 15:29:43 +0200 Subject: [PATCH 14/19] docs: update readme --- README.md | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7924fb9a82..485930431f 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,12 @@ const MulticastDNS = require('libp2p-mdns') const DHT = require('libp2p-kad-dht') const defaultsDeep = require('@nodeutils/defaults-deep') const Protector = require('libp2p-pnet') +const DelegatedPeerRouter = require('libp2p-delegated-peer-routing') +const DelegatedContentRouter = require('libp2p-delegated-content-routing') class Node extends libp2p { constructor (_options) { + const peerInfo = _options.peerInfo const defaults = { // The libp2p modules for this libp2p bundle modules: { @@ -133,8 +136,16 @@ class Node extends libp2p { connEncryption: [ SECIO ], - // Encryption for private networks. Needs additional private key to work + /** Encryption for private networks. Needs additional private key to work **/ // connProtector: new Protector(/*protector specific opts*/), + /** Enable custom content routers, such as delegated routing **/ + // contentRouting: [ + // new DelegatedContentRouter(peerInfo.id) + // ], + /** Enable custom peer routers, such as delegated routing **/ + // peerRouting: [ + // new DelegatedPeerRouter() + // ], peerDiscovery: [ MulticastDNS ], @@ -230,16 +241,17 @@ Required keys in the `options` object: `callback` is a function with the following `function (err) {}` signature, where `err` is an Error in case stopping the node fails. -#### `libp2p.peerRouting.findPeer(id, callback)` +#### `libp2p.peerRouting.findPeer(id, timeout, callback)` > Looks up for multiaddrs of a peer in the DHT - `id`: instance of [PeerId][] +- `timeout`: Number milliseconds #### `libp2p.contentRouting.findProviders(key, timeout, callback)` - `key`: Buffer -- `timeout`: Number miliseconds +- `timeout`: Number milliseconds #### `libp2p.contentRouting.provide(key, callback)` @@ -307,14 +319,16 @@ Required keys in the `options` object: - `key`: Buffer - `value`: Buffer -#### `libp2p.dht.get(key, callback)` +#### `libp2p.dht.get(key, maxTimeout, callback)` - `key`: Buffer +- `maxTimeout`: Number milliseconds -#### `libp2p.dht.getMany(key, nVals, callback)` +#### `libp2p.dht.getMany(key, nVals, maxTimeout, callback)` - `key`: Buffer - `nVals`: Number +- `maxTimeout`: Number milliseconds [PeerInfo]: https://github.com/libp2p/js-peer-info [PeerId]: https://github.com/libp2p/js-peer-id From d8d3eaaa48bc3d2a704ff5bee4670e7058be6a2b Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Thu, 27 Sep 2018 13:15:40 +0200 Subject: [PATCH 15/19] fix: update router/dht timeout param to be options --- README.md | 20 ++++++++++++-------- package.json | 4 ++-- src/content-routing.js | 16 +++++++++++++--- src/peer-routing.js | 13 +++++++------ test/peer-routing.node.js | 2 +- 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 485930431f..f86f61c495 100644 --- a/README.md +++ b/README.md @@ -241,17 +241,19 @@ Required keys in the `options` object: `callback` is a function with the following `function (err) {}` signature, where `err` is an Error in case stopping the node fails. -#### `libp2p.peerRouting.findPeer(id, timeout, callback)` +#### `libp2p.peerRouting.findPeer(id, options, callback)` > Looks up for multiaddrs of a peer in the DHT - `id`: instance of [PeerId][] -- `timeout`: Number milliseconds +- `options`: object of options +- `options.maxTimeout`: Number milliseconds -#### `libp2p.contentRouting.findProviders(key, timeout, callback)` +#### `libp2p.contentRouting.findProviders(key, options, callback)` - `key`: Buffer -- `timeout`: Number milliseconds +- `options`: object of options +- `options.maxTimeout`: Number milliseconds #### `libp2p.contentRouting.provide(key, callback)` @@ -319,16 +321,18 @@ Required keys in the `options` object: - `key`: Buffer - `value`: Buffer -#### `libp2p.dht.get(key, maxTimeout, callback)` +#### `libp2p.dht.get(key, options, callback)` - `key`: Buffer -- `maxTimeout`: Number milliseconds +- `options`: object of options +- `options.maxTimeout`: Number milliseconds -#### `libp2p.dht.getMany(key, nVals, maxTimeout, callback)` +#### `libp2p.dht.getMany(key, nVals, options, callback)` - `key`: Buffer - `nVals`: Number -- `maxTimeout`: Number milliseconds +- `options`: object of options +- `options.maxTimeout`: Number milliseconds [PeerInfo]: https://github.com/libp2p/js-peer-info [PeerId]: https://github.com/libp2p/js-peer-id diff --git a/package.json b/package.json index dde1ab52f0..beb7750d7a 100644 --- a/package.json +++ b/package.json @@ -60,8 +60,8 @@ "dirty-chai": "^2.0.1", "electron-webrtc": "~0.3.0", "libp2p-circuit": "~0.2.1", - "libp2p-delegated-content-routing": "github:libp2p/js-libp2p-delegated-content-routing#feat/timeout-defaults", - "libp2p-delegated-peer-routing": "github:libp2p/js-libp2p-delegated-peer-routing#feat/timeout-defaults", + "libp2p-delegated-content-routing": "~0.2.2", + "libp2p-delegated-peer-routing": "~0.2.2", "libp2p-kad-dht": "~0.10.5", "libp2p-mdns": "~0.12.0", "libp2p-mplex": "~0.8.2", diff --git a/src/content-routing.js b/src/content-routing.js index 5a18dbffbd..85194409d8 100644 --- a/src/content-routing.js +++ b/src/content-routing.js @@ -17,17 +17,27 @@ module.exports = (node) => { * Once a content router succeeds, iteration will stop. * * @param {CID} key The CID key of the content to find - * @param {number} timeout How long the query should run + * @param {object} options + * @param {number} options.maxTimeout How long the query should run * @param {function(Error, Result)} callback * @returns {void} */ - findProviders: (key, timeout, callback) => { + findProviders: (key, options, callback) => { if (routers.length === 0) { return callback(new Error('No content routers available')) } + if (typeof options === 'function') { + callback = options + options = {} + } else if (typeof options === 'number') { + options = { + maxTimeout: options + } + } + const tasks = routers.map((router) => { - return (cb) => router.findProviders(key, timeout, (err, results) => { + return (cb) => router.findProviders(key, options, (err, results) => { if (err) { return cb(err) } diff --git a/src/peer-routing.js b/src/peer-routing.js index 066cf31b3d..b7495cc569 100644 --- a/src/peer-routing.js +++ b/src/peer-routing.js @@ -15,22 +15,23 @@ module.exports = (node) => { * Iterates over all peer routers in series to find the given peer. * * @param {String} id The id of the peer to find - * @param {number} timeout How long the query should run + * @param {object} options + * @param {number} options.maxTimeout How long the query should run * @param {function(Error, Result)} callback * @returns {void} */ - findPeer: (id, timeout, callback) => { + findPeer: (id, options, callback) => { if (routers.length === 0) { return callback(new Error('No peer routers available')) } - if (typeof timeout === 'function') { - callback = timeout - timeout = null + if (typeof options === 'function') { + callback = options + options = {} } const tasks = routers.map((router) => { - return (cb) => router.findPeer(id, timeout, (err, result) => { + return (cb) => router.findPeer(id, options, (err, result) => { if (err) { return cb(err) } diff --git a/test/peer-routing.node.js b/test/peer-routing.node.js index 7a4cc167e9..7d728ccd9d 100644 --- a/test/peer-routing.node.js +++ b/test/peer-routing.node.js @@ -231,7 +231,7 @@ describe('.peerRouting', () => { after((done) => nodeA.stop(done)) describe('findPeer', () => { - it('should only use the dht if it find the peer', (done) => { + it('should only use the dht if it finds the peer', (done) => { const results = [true] const dhtStub = sinon.stub(nodeA._dht, 'findPeer').callsArgWith(2, null, results) const delegateStub = sinon.stub(delegate, 'findPeer').throws(() => { From 03dc65263286cab82d1459724c15bc3e1fc6ab31 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Thu, 27 Sep 2018 16:32:50 +0200 Subject: [PATCH 16/19] docs: add delegated routing example --- examples/delegated-routing/README.md | 49 ++++++ examples/delegated-routing/package.json | 26 +++ examples/delegated-routing/public/favicon.ico | Bin 0 -> 3870 bytes examples/delegated-routing/public/index.html | 15 ++ examples/delegated-routing/src/App.css | 61 +++++++ examples/delegated-routing/src/App.js | 149 ++++++++++++++++++ examples/delegated-routing/src/App.test.js | 9 ++ examples/delegated-routing/src/index.css | 5 + examples/delegated-routing/src/index.js | 6 + .../delegated-routing/src/libp2p-bundle.js | 75 +++++++++ 10 files changed, 395 insertions(+) create mode 100644 examples/delegated-routing/README.md create mode 100644 examples/delegated-routing/package.json create mode 100644 examples/delegated-routing/public/favicon.ico create mode 100644 examples/delegated-routing/public/index.html create mode 100644 examples/delegated-routing/src/App.css create mode 100644 examples/delegated-routing/src/App.js create mode 100644 examples/delegated-routing/src/App.test.js create mode 100644 examples/delegated-routing/src/index.css create mode 100644 examples/delegated-routing/src/index.js create mode 100644 examples/delegated-routing/src/libp2p-bundle.js diff --git a/examples/delegated-routing/README.md b/examples/delegated-routing/README.md new file mode 100644 index 0000000000..debd15407f --- /dev/null +++ b/examples/delegated-routing/README.md @@ -0,0 +1,49 @@ +# Delegated Routing with Libp2p and IPFS + +This example shows how to use delegated peer and content routing. The [Peer and Content Routing Example](../peer-and-content-routing) focuses +on the DHT implementation. This example takes that a step further and introduces delegated routing. Delegated routing is +especially useful when your libp2p node will have limited resources, making running a DHT impractical. It's +also highly useful if your node is generating content, but can't reliably be on the network. You can use delegate nodes +to provide content on your behalf. + +The starting [Libp2p Bundle](./src/libp2p-bundle.js) in this example starts by disabling the DHT and adding the Delegated Peer and Content Routers. +Once you've completed the example, you should try enabled the DHT and see what kind of results you get! You can also enable the +various Peer Discovery modules and see the impact it has on your Peer count. + +## Prerequisite +**NOTE**: This example is currently dependent on a clone of the [delegated routing support branch of go-ipfs](https://github.com/ipfs/go-ipfs/pull/4595). + +## Running this example + +1. Install IPFS locally if you dont already have it. [Install Guide](https://docs.ipfs.io/introduction/install/) +2. Run the IPFS daemon: `ipfs daemon` +3. The daemon will output a line about its API address, like `API server listening on /ip4/127.0.0.1/tcp/8080` +4. In another window output the addresses of the node: `ipfs id`. Make note of the websocket address, is will contain `/ws/` in the address. +5. In `./src/libp2p-bundle.js` replace the `delegatedApiOptions` host and port of your node if they are different. +6. In `./src/App.js` replace `BootstrapNode` with your nodes Websocket address from step 4. +7. Start this example: + +```sh +npm install +npm start +``` + +This should open your browser to http://localhost:3000. If it does not, go ahead and do that now. + +8. Your browser should show you connected to at least 1 peer. + +### Finding Content via the Delegate +1. Add a file to your IPFS node. From this example root you can do `ipfs add ./README.md` to add the example readme. +2. Copy the hash from line 5, it will look something like *Qmf33vz4HJFkqgH7XPP1uA6atYKTX1BWQEQthzpKcAdeyZ*. +3. In the browser, paste the hash into the *Hash* field and hit `Find`. The readme contents should display. + +This will do a few things: +* The delegate nodes api will be queried to find providers of the content +* The content will be fetched from the providers +* Since we now have the content, we tell the delegate node to fetch the content from us and become a provider + +### Finding Peers via the Delegate +1. Get a list of your delegate nodes peer by querying the IPFS daemon: `ipfs swarm peers` +2. Copy one of the CIDs from the list of peer addresses, this will be the last portion of the address and will look something like `QmdoG8DpzYUZMVP5dGmgmigZwR1RE8Cf6SxMPg1SBXJAQ8`. +3. In your browser, paste the CID into the *Peer* field and hit `Find`. +4. You should see information about the peer including its addresses. diff --git a/examples/delegated-routing/package.json b/examples/delegated-routing/package.json new file mode 100644 index 0000000000..649171c544 --- /dev/null +++ b/examples/delegated-routing/package.json @@ -0,0 +1,26 @@ +{ + "name": "delegated-routing-example", + "version": "0.1.0", + "private": true, + "dependencies": { + "ipfs": "~0.32.2", + "libp2p": "../../", + "libp2p-delegated-content-routing": "~0.2.2", + "libp2p-delegated-peer-routing": "~0.2.2", + "libp2p-kad-dht": "~0.10.4", + "libp2p-mplex": "~0.8.0", + "libp2p-secio": "~0.10.0", + "libp2p-webrtc-star": "~0.15.5", + "libp2p-websocket-star": "~0.8.1", + "libp2p-websockets": "~0.12.0", + "react": "^16.5.2", + "react-dom": "^16.5.2", + "react-scripts": "1.1.5" + }, + "scripts": { + "start": "react-scripts start", + "build": "react-scripts build", + "test": "react-scripts test --env=jsdom", + "eject": "react-scripts eject" + } +} diff --git a/examples/delegated-routing/public/favicon.ico b/examples/delegated-routing/public/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..a11777cc471a4344702741ab1c8a588998b1311a GIT binary patch literal 3870 zcma);c{J4h9>;%nil|2-o+rCuEF-(I%-F}ijC~o(k~HKAkr0)!FCj~d>`RtpD?8b; zXOC1OD!V*IsqUwzbMF1)-gEDD=A573Z-&G7^LoAC9|WO7Xc0Cx1g^Zu0u_SjAPB3vGa^W|sj)80f#V0@M_CAZTIO(t--xg= z!sii`1giyH7EKL_+Wi0ab<)&E_0KD!3Rp2^HNB*K2@PHCs4PWSA32*-^7d{9nH2_E zmC{C*N*)(vEF1_aMamw2A{ZH5aIDqiabnFdJ|y0%aS|64E$`s2ccV~3lR!u<){eS` z#^Mx6o(iP1Ix%4dv`t@!&Za-K@mTm#vadc{0aWDV*_%EiGK7qMC_(`exc>-$Gb9~W!w_^{*pYRm~G zBN{nA;cm^w$VWg1O^^<6vY`1XCD|s_zv*g*5&V#wv&s#h$xlUilPe4U@I&UXZbL z0)%9Uj&@yd03n;!7do+bfixH^FeZ-Ema}s;DQX2gY+7g0s(9;`8GyvPY1*vxiF&|w z>!vA~GA<~JUqH}d;DfBSi^IT*#lrzXl$fNpq0_T1tA+`A$1?(gLb?e#0>UELvljtQ zK+*74m0jn&)5yk8mLBv;=@}c{t0ztT<v;Avck$S6D`Z)^c0(jiwKhQsn|LDRY&w(Fmi91I7H6S;b0XM{e zXp0~(T@k_r-!jkLwd1_Vre^v$G4|kh4}=Gi?$AaJ)3I+^m|Zyj#*?Kp@w(lQdJZf4 z#|IJW5z+S^e9@(6hW6N~{pj8|NO*>1)E=%?nNUAkmv~OY&ZV;m-%?pQ_11)hAr0oAwILrlsGawpxx4D43J&K=n+p3WLnlDsQ$b(9+4 z?mO^hmV^F8MV{4Lx>(Q=aHhQ1){0d*(e&s%G=i5rq3;t{JC zmgbn5Nkl)t@fPH$v;af26lyhH!k+#}_&aBK4baYPbZy$5aFx4}ka&qxl z$=Rh$W;U)>-=S-0=?7FH9dUAd2(q#4TCAHky!$^~;Dz^j|8_wuKc*YzfdAht@Q&ror?91Dm!N03=4=O!a)I*0q~p0g$Fm$pmr$ zb;wD;STDIi$@M%y1>p&_>%?UP($15gou_ue1u0!4(%81;qcIW8NyxFEvXpiJ|H4wz z*mFT(qVx1FKufG11hByuX%lPk4t#WZ{>8ka2efjY`~;AL6vWyQKpJun2nRiZYDij$ zP>4jQXPaP$UC$yIVgGa)jDV;F0l^n(V=HMRB5)20V7&r$jmk{UUIe zVjKroK}JAbD>B`2cwNQ&GDLx8{pg`7hbA~grk|W6LgiZ`8y`{Iq0i>t!3p2}MS6S+ zO_ruKyAElt)rdS>CtF7j{&6rP-#c=7evGMt7B6`7HG|-(WL`bDUAjyn+k$mx$CH;q2Dz4x;cPP$hW=`pFfLO)!jaCL@V2+F)So3}vg|%O*^T1j>C2lx zsURO-zIJC$^$g2byVbRIo^w>UxK}74^TqUiRR#7s_X$e)$6iYG1(PcW7un-va-S&u zHk9-6Zn&>T==A)lM^D~bk{&rFzCi35>UR!ZjQkdSiNX*-;l4z9j*7|q`TBl~Au`5& z+c)*8?#-tgUR$Zd%Q3bs96w6k7q@#tUn`5rj+r@_sAVVLqco|6O{ILX&U-&-cbVa3 zY?ngHR@%l{;`ri%H*0EhBWrGjv!LE4db?HEWb5mu*t@{kv|XwK8?npOshmzf=vZA@ zVSN9sL~!sn?r(AK)Q7Jk2(|M67Uy3I{eRy z_l&Y@A>;vjkWN5I2xvFFTLX0i+`{qz7C_@bo`ZUzDugfq4+>a3?1v%)O+YTd6@Ul7 zAfLfm=nhZ`)P~&v90$&UcF+yXm9sq!qCx3^9gzIcO|Y(js^Fj)Rvq>nQAHI92ap=P z10A4@prk+AGWCb`2)dQYFuR$|H6iDE8p}9a?#nV2}LBCoCf(Xi2@szia7#gY>b|l!-U`c}@ zLdhvQjc!BdLJvYvzzzngnw51yRYCqh4}$oRCy-z|v3Hc*d|?^Wj=l~18*E~*cR_kU z{XsxM1i{V*4GujHQ3DBpl2w4FgFR48Nma@HPgnyKoIEY-MqmMeY=I<%oG~l!f<+FN z1ZY^;10j4M4#HYXP zw5eJpA_y(>uLQ~OucgxDLuf}fVs272FaMxhn4xnDGIyLXnw>Xsd^J8XhcWIwIoQ9} z%FoSJTAGW(SRGwJwb=@pY7r$uQRK3Zd~XbxU)ts!4XsJrCycrWSI?e!IqwqIR8+Jh zlRjZ`UO1I!BtJR_2~7AbkbSm%XQqxEPkz6BTGWx8e}nQ=w7bZ|eVP4?*Tb!$(R)iC z9)&%bS*u(lXqzitAN)Oo=&Ytn>%Hzjc<5liuPi>zC_nw;Z0AE3Y$Jao_Q90R-gl~5 z_xAb2J%eArrC1CN4G$}-zVvCqF1;H;abAu6G*+PDHSYFx@Tdbfox*uEd3}BUyYY-l zTfEsOqsi#f9^FoLO;ChK<554qkri&Av~SIM*{fEYRE?vH7pTAOmu2pz3X?Wn*!ROX ztd54huAk&mFBemMooL33RV-*1f0Q3_(7hl$<#*|WF9P!;r;4_+X~k~uKEqdzZ$5Al zV63XN@)j$FN#cCD;ek1R#l zv%pGrhB~KWgoCj%GT?%{@@o(AJGt*PG#l3i>lhmb_twKH^EYvacVY-6bsCl5*^~L0 zonm@lk2UvvTKr2RS%}T>^~EYqdL1q4nD%0n&Xqr^cK^`J5W;lRRB^R-O8b&HENO||mo0xaD+S=I8RTlIfVgqN@SXDr2&-)we--K7w= zJVU8?Z+7k9dy;s;^gDkQa`0nz6N{T?(A&Iz)2!DEecLyRa&FI!id#5Z7B*O2=PsR0 zEvc|8{NS^)!d)MDX(97Xw}m&kEO@5jqRaDZ!+%`wYOI<23q|&js`&o4xvjP7D_xv@ z5hEwpsp{HezI9!~6O{~)lLR@oF7?J7i>1|5a~UuoN=q&6N}EJPV_GD`&M*v8Y`^2j zKII*d_@Fi$+i*YEW+Hbzn{iQk~yP z>7N{S4)r*!NwQ`(qcN#8SRQsNK6>{)X12nbF`*7#ecO7I)Q$uZsV+xS4E7aUn+U(K baj7?x%VD!5Cxk2YbYLNVeiXvvpMCWYo=by@ literal 0 HcmV?d00001 diff --git a/examples/delegated-routing/public/index.html b/examples/delegated-routing/public/index.html new file mode 100644 index 0000000000..405ec11100 --- /dev/null +++ b/examples/delegated-routing/public/index.html @@ -0,0 +1,15 @@ + + + + + + + Delegated Routing + + + +
+ + diff --git a/examples/delegated-routing/src/App.css b/examples/delegated-routing/src/App.css new file mode 100644 index 0000000000..a050008a78 --- /dev/null +++ b/examples/delegated-routing/src/App.css @@ -0,0 +1,61 @@ +section * { + margin: 10px; +} + +header { + background-color: #222; + height: 150px; + padding: 20px; + color: white; +} + +.center { + text-align: center; +} + +pre { + background-color: bisque; + min-height: 100px; + margin: 0px; + padding: 10px; +} + +.loader { + text-align: center; + height: 64px; + margin-bottom: -64px; +} + +.loading .lds-ripple { + display: inline-block; + position: relative; + width: 64px; + height: 64px; +} +.loading .lds-ripple div { + position: absolute; + border: 4px solid #000; + opacity: 1; + border-radius: 50%; + animation: lds-ripple 1s cubic-bezier(0, 0.2, 0.8, 1) infinite; + margin: auto; +} +.loading .lds-ripple div:nth-child(2) { + animation-delay: -0.5s; +} +@keyframes lds-ripple { + 0% { + top: 28px; + left: 28px; + width: 0; + height: 0; + opacity: 1; + } + 100% { + top: -1px; + left: -1px; + width: 58px; + height: 58px; + opacity: 0; + } +} \ No newline at end of file diff --git a/examples/delegated-routing/src/App.js b/examples/delegated-routing/src/App.js new file mode 100644 index 0000000000..f92ec0dd3b --- /dev/null +++ b/examples/delegated-routing/src/App.js @@ -0,0 +1,149 @@ +import React, { Component } from 'react'; +import './App.css'; +import Ipfs from 'ipfs'; +import libp2pBundle from './libp2p-bundle'; + +const BootstrapNode = '/ip4/127.0.0.1/tcp/8081/ws/ipfs/QmdoG8DpzYUZMVP5dGmgmigZwR1RE8Cf6SxMPg1SBXJAQ8' + +class App extends Component { + constructor(props) { + super(props); + this.state = { + peers: 0, + // This hash is the IPFS readme + hash: 'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB', + // This peer is one of the Bootstrap nodes for IPFS + peer: 'QmV6kA2fB8kTr6jc3pL5zbNsjKbmPUHAPKKHRBYe1kDEyc', + isLoading: 0 + }; + this.peerInterval = null; + + this.handleHashChange = this.handleHashChange.bind(this); + this.handleHashSubmit = this.handleHashSubmit.bind(this); + this.handlePeerChange = this.handlePeerChange.bind(this); + this.handlePeerSubmit = this.handlePeerSubmit.bind(this); + } + + handleHashChange(event) { + this.setState({ + hash: event.target.value + }); + } + handlePeerChange(event) { + this.setState({ + peer: event.target.value + }); + } + + handleHashSubmit(event) { + event.preventDefault(); + this.setState({ + isLoading: this.state.isLoading + 1 + }) + + this.ipfs.files.cat(this.state.hash, (err, data) => { + if (err) console.log('Error', err) + + this.setState({ + response: data.toString(), + isLoading: this.state.isLoading - 1 + }) + }); + } + handlePeerSubmit(event) { + event.preventDefault(); + this.setState({ + isLoading: this.state.isLoading + 1 + }) + + this.ipfs.dht.findpeer(this.state.peer, (err, results) => { + if (err) console.log('Error', err) + + this.setState({ + response: JSON.stringify(results, null, 2), + isLoading: this.state.isLoading - 1 + }) + }); + } + + componentDidMount() { + window.ipfs = this.ipfs = new Ipfs({ + config: { + Addresses: { + Swarm: [] + }, + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + }, + Bootstrap: [ + BootstrapNode + ] + }, + preload: { + enabled: false + }, + libp2p: libp2pBundle + }) + this.ipfs.on('ready', () => { + if (this.peerInterval) { + clearInterval(this.peerInterval) + } + + this.ipfs.swarm.connect(BootstrapNode, (err) => { + if (err) { + console.log('Error connecting to the node', err) + } + console.log('Connected!') + }) + + this.peerInterval = setInterval(() => { + this.ipfs.swarm.peers((err, peers) => { + if (err) console.log(err) + if (peers) this.setState({peers: peers.length}) + }) + }, 2500) + }) + } + + render() { + return ( +
+
+

Delegated Routing

+

There are currently {this.state.peers} peers.

+
+
+
+ +
+
+ +
+
+
0 ? 'loading': '', 'loader'].join(' ')}> +
+
+
+ 
+            {this.state.response}
+
+
+
+ ); + } +} + +export default App; diff --git a/examples/delegated-routing/src/App.test.js b/examples/delegated-routing/src/App.test.js new file mode 100644 index 0000000000..a754b201bf --- /dev/null +++ b/examples/delegated-routing/src/App.test.js @@ -0,0 +1,9 @@ +import React from 'react'; +import ReactDOM from 'react-dom'; +import App from './App'; + +it('renders without crashing', () => { + const div = document.createElement('div'); + ReactDOM.render(, div); + ReactDOM.unmountComponentAtNode(div); +}); diff --git a/examples/delegated-routing/src/index.css b/examples/delegated-routing/src/index.css new file mode 100644 index 0000000000..b4cc7250b9 --- /dev/null +++ b/examples/delegated-routing/src/index.css @@ -0,0 +1,5 @@ +body { + margin: 0; + padding: 0; + font-family: sans-serif; +} diff --git a/examples/delegated-routing/src/index.js b/examples/delegated-routing/src/index.js new file mode 100644 index 0000000000..395b74997b --- /dev/null +++ b/examples/delegated-routing/src/index.js @@ -0,0 +1,6 @@ +import React from 'react'; +import ReactDOM from 'react-dom'; +import './index.css'; +import App from './App'; + +ReactDOM.render(, document.getElementById('root')); diff --git a/examples/delegated-routing/src/libp2p-bundle.js b/examples/delegated-routing/src/libp2p-bundle.js new file mode 100644 index 0000000000..12c6792f1b --- /dev/null +++ b/examples/delegated-routing/src/libp2p-bundle.js @@ -0,0 +1,75 @@ +import Libp2p from 'libp2p'; +import Websockets from 'libp2p-websockets'; +import WebSocketStar from 'libp2p-websocket-star'; +import WebRTCStar from 'libp2p-webrtc-star'; +import MPLEX from 'libp2p-mplex'; +import SECIO from 'libp2p-secio'; +import KadDHT from 'libp2p-kad-dht'; +import DelegatedPeerRouter from 'libp2p-delegated-peer-routing'; +import DelegatedContentRouter from 'libp2p-delegated-content-routing'; + +export default ({peerInfo, peerBook}) => { + const wrtcstar = new WebRTCStar({id: peerInfo.id}); + const wsstar = new WebSocketStar({id: peerInfo.id}); + const delegatedApiOptions = { + host: '0.0.0.0', + protocol: 'http', + port: '8080' + } + + return new Libp2p({ + peerInfo, + peerBook, + // Lets limit the connection managers peers and have it check peer health less frequently + connectionManager: { + maxPeers: 10, + pollInterval: 5000 + }, + modules: { + contentRouting: [ + new DelegatedContentRouter(peerInfo.id, delegatedApiOptions) + ], + peerRouting: [ + new DelegatedPeerRouter(delegatedApiOptions) + ], + peerDiscovery: [ + wrtcstar.discovery, + wsstar.discovery + ], + transport: [ + wrtcstar, + wsstar, + Websockets + ], + streamMuxer: [ + MPLEX + ], + connEncryption: [ + SECIO + ], + dht: KadDHT + }, + config: { + peerDiscovery: { + webrtcStar: { + enabled: false + }, + websocketStar: { + enabled: false + } + }, + dht: { + kBucketSize: 20 + }, + relay: { + enabled: true, + hop: { + enabled: false + } + }, + EXPERIMENTAL: { + dht: false + } + } + }) +}; \ No newline at end of file From b84ba05f68f1f6bdbbb36c90a6d7bc29c321a0a4 Mon Sep 17 00:00:00 2001 From: Jacob Heun Date: Fri, 28 Sep 2018 12:07:56 +0200 Subject: [PATCH 17/19] fix(docs): linting --- examples/delegated-routing/package.json | 5 +- examples/delegated-routing/public/index.html | 1 + .../{src/App.css => public/main.css} | 6 ++ examples/delegated-routing/src/App.js | 60 ++++++++++--------- examples/delegated-routing/src/App.test.js | 9 --- examples/delegated-routing/src/index.css | 5 -- examples/delegated-routing/src/index.js | 13 ++-- .../delegated-routing/src/libp2p-bundle.js | 29 +++++---- 8 files changed, 64 insertions(+), 64 deletions(-) rename examples/delegated-routing/{src/App.css => public/main.css} (93%) delete mode 100644 examples/delegated-routing/src/App.test.js delete mode 100644 examples/delegated-routing/src/index.css diff --git a/examples/delegated-routing/package.json b/examples/delegated-routing/package.json index 649171c544..ff36b7bd79 100644 --- a/examples/delegated-routing/package.json +++ b/examples/delegated-routing/package.json @@ -18,9 +18,6 @@ "react-scripts": "1.1.5" }, "scripts": { - "start": "react-scripts start", - "build": "react-scripts build", - "test": "react-scripts test --env=jsdom", - "eject": "react-scripts eject" + "start": "react-scripts start" } } diff --git a/examples/delegated-routing/public/index.html b/examples/delegated-routing/public/index.html index 405ec11100..9082175b4e 100644 --- a/examples/delegated-routing/public/index.html +++ b/examples/delegated-routing/public/index.html @@ -5,6 +5,7 @@ Delegated Routing +