From b28edc8e38968bf6fc3b09d69da0d5001ea35daa Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 11 Jan 2021 15:57:02 +0100 Subject: [PATCH] chore: add benchmarks --- .aegir.js | 27 +---- benchmarks/README.md | 108 ++++++++++++++++++ benchmarks/id.json | 5 + benchmarks/index.js | 248 ++++++++++++++++++++++++++++++++++++++++++ benchmarks/run.sh | 18 +++ benchmarks/utils.js | 21 ++++ mysql-local/docker.js | 33 ++++++ package.json | 6 +- src/bin.js | 1 + 9 files changed, 442 insertions(+), 25 deletions(-) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/id.json create mode 100644 benchmarks/index.js create mode 100755 benchmarks/run.sh create mode 100644 benchmarks/utils.js create mode 100644 mysql-local/docker.js diff --git a/.aegir.js b/.aegir.js index 707f042..5e4bd8a 100644 --- a/.aegir.js +++ b/.aegir.js @@ -3,14 +3,12 @@ const Libp2p = require('libp2p') const { MULTIADDRS_WEBSOCKETS } = require('./test/fixtures/browser') const Peers = require('./test/fixtures/peers') +const docker = require('./mysql-local/docker') const PeerId = require('peer-id') const WebSockets = require('libp2p-websockets') const Muxer = require('libp2p-mplex') const { NOISE: Crypto } = require('libp2p-noise') -const delay = require('delay') -const execa = require('execa') -const pWaitFor = require('p-wait-for') const isCI = require('is-ci') let libp2p @@ -50,34 +48,17 @@ const before = async () => { return } - const procResult = execa.commandSync('docker run -p 3306:3306 -e MYSQL_ROOT_PASSWORD=test-secret-pw -e MYSQL_DATABASE=libp2p_rendezvous_db -d mysql:8 --default-authentication-plugin=mysql_native_password', { - all: true - }) - containerId = procResult.stdout - - console.log(`wait for docker container ${containerId} to be ready`) - - await pWaitFor(() => { - const procCheck = execa.commandSync(`docker logs ${containerId}`) - const logs = procCheck.stdout + procCheck.stderr // Docker/MySQL sends to the stderr the ready for connections... - - return logs.includes('ready for connections') - }, { - interval: 5000 - }) - // Some more time waiting to guarantee the container is really ready - await delay(12e3) + containerId = await docker.start() } const after = async () => { await libp2p.stop() - if (isCI) { + if (isCI || !containerId) { return } - console.log('docker container is stopping') - execa.commandSync(`docker stop ${containerId}`) + docker.stop(containerId) } module.exports = { diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..65fe0b8 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,108 @@ +# Rendezvous benchmarks + +This benchmark contains a simulator to exercise a rendezvous server and gather performance metrics from it. + +## Running + +For running the benchmarks, it is required to install the dependencies of the `libp2p-rendezvous`, as well as Docker. With those installed, you only need to run the `index.js` file as follows: + +```sh +$ npm install +$ cd benchmarks +$ node index.js +``` + +While default values exist for benchmarking, you can use CLI parameters to configure how to run the benchmark. + +### Configuration + +TODO + +### Output + +TODO + +## Metrics + +- Operations {Register, Discover} + - Average response time + - Maximum response time + - Median response time +- Server performance + - CPU + - Memory + +## Scenarios + +### Register + +Measure adding n registrations. Each operation in the following table + +| Type | Clients | Io registrations | Operations | Namespaces | +|------|---------|------------------|------------|------------| +| `Register` | 5 | 0 | 500 | 10 | +| `Register` | 5 | 1000 | 500 | 10 | +| `Register` | 10 | 1000 | 500 | 10 | +| `Register` | 100 | 1000 | 500 | 10 | +| `Register` | 100 | 1000 | 1000 | 10 | +| `Register` | 100 | 10000 | 500 | 10 | +| `Register` | 100 | 100000 | 1000 | 10 | +| `Register` | 100 | 100000 | 10000 | 10 | +| `Register` | 1000 | 100000 | 10000 | 10 | +| `Register` | 1000 | 100000 | 100000 | 10 | +| `Register` | 1000 | 1000000 | 10000 | 10 | +| `Register` | 1000 | 1000000 | 100000 | 10 | +| `Register` | 1000 | 10000000 | 10000 | 10 | +| `Register` | 1000 | 10000000 | 100000 | 10 | +| `Register` | 1000 | 100000000 | 10000 | 10 | +| `Register` | 1000 | 100000000 | 100000 | 10 | + +### Discover + +1. Measure discover 500 existing registrations in series with limit of 20 + +| Type | Io registrations | Io namespaces | +|------|------------------|------------------| +| `Discover` | 0 | 0 | +| `Discover` | 100 | 10 | +| `Discover` | 1000 | 10 | +| `Discover` | 1000 | 100 | +| `Discover` | 10000 | 10 | +| `Discover` | 10000 | 100 | +| `Discover` | 10000 | 1000 | +| `Discover` | 100000 | 10000 | +| `Discover` | 1000000 | 10 | +| `Discover` | 1000000 | 100 | +| `Discover` | 1000000 | 1000 | +| `Discover` | 1000000 | 10000 | +| `Discover` | 1000000 | 100000 | +| `Discover` | 10000000 | 100000 | +| `Discover` | 100000000 | 100000 | + +2. Measure trying to discover 500 not existing registrations in series + +| Type | Io registrations | Io namespaces | +|------|------------------|------------------| +| `Discover` | 0 | 0 | +| `Discover` | 100 | 10 | +| `Discover` | 1000 | 10 | +| `Discover` | 1000 | 100 | +| `Discover` | 10000 | 10 | +| `Discover` | 10000 | 100 | +| `Discover` | 10000 | 1000 | +| `Discover` | 100000 | 10000 | +| `Discover` | 1000000 | 10 | +| `Discover` | 1000000 | 100 | +| `Discover` | 1000000 | 1000 | +| `Discover` | 1000000 | 10000 | +| `Discover` | 1000000 | 100000 | +| `Discover` | 10000000 | 100000 | +| `Discover` | 100000000 | 100000 | + +3. Cookies + + + +| Type | Io Reg | Namespaces | Ops | Average RT (ms) | Median RT (ms) | Max RT (ms) | Min RT (ms) | +|----------|--------|------------|-----|-----------------|----------------|-------------|-------------| +| REGISTER | 100 | 500 | 5 | 1243 | 1228 | 2456 | 24 | \ No newline at end of file diff --git a/benchmarks/id.json b/benchmarks/id.json new file mode 100644 index 0000000..1373524 --- /dev/null +++ b/benchmarks/id.json @@ -0,0 +1,5 @@ +{ + "id": "12D3KooWPCDhKA2NoJCnxUqKmBNLVg6ekNzcw9GVsncLgfdbN2pm", + "privKey": "CAESYDprk82zAJeNcIHxgj3seEWRLCOh+7e7yTBVx1IW38HnxsEgoQGbW5xJDd7GHml2Mb8LNxsdB+WgznkhDLYZL97GwSChAZtbnEkN3sYeaXYxvws3Gx0H5aDOeSEMthkv3g==", + "pubKey": "CAESIMbBIKEBm1ucSQ3exh5pdjG/CzcbHQfloM55IQy2GS/e" +} diff --git a/benchmarks/index.js b/benchmarks/index.js new file mode 100644 index 0000000..c6df0ee --- /dev/null +++ b/benchmarks/index.js @@ -0,0 +1,248 @@ +'use strict' + +/* eslint-disable no-console */ + +const fs = require('fs') +const execa = require('execa') +const argv = require('minimist')(process.argv.slice(2)) +const microtime = require('microtime') +const path = require('path') +const pidusage = require('pidusage') +const pDefer = require('p-defer') +const uint8ArrayToString = require('uint8arrays/to-string') + +const { pipe } = require('it-pipe') +const lp = require('it-length-prefixed') +const { + collect, + tap +} = require('streaming-iterables') +const { toBuffer } = require('it-buffer') + +const Libp2p = require('libp2p') +const PeerId = require('peer-id') + +const docker = require('../mysql-local/docker') +const ServerPeerId = require('./id.json') +const { median } = require('./utils') + +const { + PROTOCOL_MULTICODEC +} = require('../src/constants') +const { Message } = require('../src/proto') +const MESSAGE_TYPE = Message.MessageType + +const { defaultLibp2pConfig } = require('../test/utils') + +/** + * Setup Rendezvous server process and multiple clients with a connection with the server. + * Outputs metrics reports on teardown. + * + * @param {number} nClients + * @returns {{ connections: Connection[], clients: Libp2p[], teardown: () => void}} + */ +const setupRendezvous = async (nClients) => { + // Setup datastore + console.log('1. Datastore setup') + const containerId = await docker.start() + + // Setup Server + console.log('2. Rendezvous Server setup') + const serverDefer = pDefer() + const serverProcess = execa('node', [path.join(__dirname, '../src/bin.js'), '--peerId', './id.json'], { + cwd: path.resolve(__dirname), + all: true + }) + serverProcess.all.on('data', (data) => { + process.stdout.write(data) + const output = uint8ArrayToString(data) + + if (output.includes('Rendezvous server listening on:')) { + serverDefer.resolve() + } + }) + + const serverProcessId = serverProcess.pid + + await serverDefer.promise + + const serverPeerId = await PeerId.createFromJSON(ServerPeerId) + const serverMultiaddr = `/ip4/127.0.0.1/tcp/15003/ws/p2p/${serverPeerId.toB58String()}` + + const clients = [] + const connections = [] + + for (let i = 0; i < nClients; i++) { + console.log(`3. Rendezvous Client ${i} setup`) + + // Setup Client + const client = await Libp2p.create({ + ...defaultLibp2pConfig, + addresses: { + listen: ['/ip4/127.0.0.1/tcp/0/ws'] + } + }) + + await client.start() + const connection = await client.dial(serverMultiaddr) + + clients.push(client) + connections.push(connection) + } + + return { + clients, + connections, + serverProcessId, + teardown: async () => { + serverProcess.kill() + + await Promise.all([ + serverProcess, + clients.map((client) => client.stop()) + ]) + + docker.stop(containerId) + } + } +} + +const createRegisterMessages = (nRuns, nClients, nNamespaces, signedPeerRecord) => { + return Array.from({ length: nRuns / nClients }, (_, runIndex) => Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord, + ns: `${runIndex % nNamespaces}`, + ttl: 10000 + } + })) +} + +const sendMessages = async (clients, connections, nRuns, nNamespaces, type = 'REGISTER') => { + let countErrors = 0 + const responseTimes = [] + + await Promise.all(Array.from({ length: clients.length }, async (_, i) => { + const startTimes = [] + const endTimes = [] + + // TODO: create source according to message type + const signedPeerRecord = clients[i].peerStore.addressBook.getRawEnvelope(clients[i].peerId) + const source = createRegisterMessages(nRuns, clients.length, nNamespaces, signedPeerRecord) + + const { stream } = await connections[i].newStream(PROTOCOL_MULTICODEC) + + const responses = await pipe( + source, + lp.encode(), + tap(() => { + startTimes.push(microtime.now()) + }), + stream, + tap(() => { + endTimes.push(microtime.now()) + }), + lp.decode(), + toBuffer, + collect + ) + + responses.forEach((r) => { + const m = Message.decode(r) + if (m.registerResponse.status !== 0) { + countErrors++ + } + }) + + startTimes.forEach((start, timeIndex) => { + responseTimes.push(endTimes[timeIndex] - start) + }) + })) + + return { + responseTimes, + countErrors + } +} + +const computePidUsage = (pid) => { + const measuremennts = [] + + const _intervalId = setInterval(() => { + pidusage(pid, (_, { cpu, memory }) => { + measuremennts.push({ cpu, memory }) + }) + }, 500) + + return { + measuremennts, + teardown: () => clearInterval(_intervalId) + } +} + +const tableHeader = `| Type | Clients | Io Reg | Namespaces | Ops | Avg RT | Median RT | Max RT | Avg CPU | Median CPU | Max CPU | Avg Mem | Median Mem | Max Mem | +|----------|---------|--------|------------|-----|--------|-----------|--------|---------|------------|---------|---------|------------|---------|` + +// Usage: $0 [--nClients ] [--nNamespaces ] [--initialRegistrations ] +// [--benchmarkRuns ] [--benchmarkType ] [--outputFile ] +const main = async () => { + const nClients = argv.nClients || 4 + const nNamespaces = argv.nNamespaces || 5 + + const initalRegistrations = argv.initialRegistrations || 100 + const benchmarkRuns = argv.benchmarkRuns || 500 + const benchmarkType = argv.benchmarkType === 'DISCOVER' ? 'DISCOVER' : 'REGISTER' + + const outputPath = argv.outputFile || './output.md' + + // Setup datastore, server and clients + console.log('==========--- Setup ---==========') + const { clients, connections, serverProcessId, teardown } = await setupRendezvous(nClients) + + // Populate Initial State and prepare data in memory + console.log('==========--- Initial State Population ---==========') + await sendMessages(clients, connections, initalRegistrations, nNamespaces) + + console.log('==========--- Start Benchmark ---==========') + const { measuremennts, teardown: pidUsageTeatdown } = computePidUsage(serverProcessId) + const { responseTimes, countErrors } = await sendMessages(clients, connections, benchmarkRuns, nNamespaces, benchmarkType) + + pidUsageTeatdown() + console.log('==========--- Finished! Compute Metrics ---==========') + + console.log('operations errored', countErrors) + + const averageRT = Math.floor(responseTimes.reduce((a, b) => a + b) / responseTimes.length / 1000) + const medianRT = Math.floor(median(responseTimes) / 1000) + const maxRT = Math.floor(Math.max(...responseTimes) / 1000) + + const cpuM = measuremennts.map((m) => m.cpu) + const averageCPU = Math.floor(cpuM.reduce((a, b) => a + b) / cpuM.length) + const medianCPU = Math.floor(median(cpuM)) + const maxCPU = Math.floor(Math.max(...cpuM)) + + const memM = measuremennts.map((m) => m.memory) + const averageMem = Math.floor(memM.reduce((a, b) => a + b) * Math.pow(10, -6) / measuremennts.length) + const medianMem = Math.floor(median(memM) * Math.pow(10, -6)) + const maxMem = Math.floor(Math.max(...memM) * Math.pow(10, -6)) + + const resultString = `| ${benchmarkType} | ${nClients} | ${initalRegistrations} | ${nNamespaces} | ${benchmarkRuns} | ${averageRT} | ${medianRT} | ${maxRT} | ${averageCPU} | ${medianCPU} | ${maxCPU} | ${averageMem} | ${medianMem} | ${maxMem} |` + + console.log(tableHeader) + console.log(resultString) + + await new Promise((resolve) => setTimeout(() => resolve(), 4000)) + await teardown() + + try { + if (fs.existsSync(outputPath)) { + fs.appendFileSync(outputPath, `\n${resultString}`) + } else { + fs.appendFileSync(outputPath, `${tableHeader}\n${resultString}`) + } + } catch (err) { + console.error(err) + } +} + +main() diff --git a/benchmarks/run.sh b/benchmarks/run.sh new file mode 100755 index 0000000..d7ab6f5 --- /dev/null +++ b/benchmarks/run.sh @@ -0,0 +1,18 @@ +# Register + +node index.js --nClients 5 --initialRegistrations 0 --benchmarkRuns 500 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 5 --initialRegistrations 1000 --benchmarkRuns 500 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 10 --initialRegistrations 1000 --benchmarkRuns 500 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 100 --initialRegistrations 1000 --benchmarkRuns 500 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 100 --initialRegistrations 1000 --benchmarkRuns 1000 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 100 --initialRegistrations 10000 --benchmarkRuns 500 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 100 --initialRegistrations 100000 --benchmarkRuns 1000 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 100 --initialRegistrations 100000 --benchmarkRuns 10000 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 1000 --initialRegistrations 100000 --benchmarkRuns 10000 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 1000 --initialRegistrations 100000 --benchmarkRuns 100000 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 1000 --initialRegistrations 1000000 --benchmarkRuns 10000 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 1000 --initialRegistrations 1000000 --benchmarkRuns 100000 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 1000 --initialRegistrations 10000000 --benchmarkRuns 10000 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 1000 --initialRegistrations 10000000 --benchmarkRuns 100000 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 1000 --initialRegistrations 100000000 --benchmarkRuns 10000 --nNamespaces 10 --outputFile './output-register.md' +node index.js --nClients 1000 --initialRegistrations 100000000 --benchmarkRuns 100000 --nNamespaces 10 --outputFile './output-register.md' diff --git a/benchmarks/utils.js b/benchmarks/utils.js new file mode 100644 index 0000000..a21d721 --- /dev/null +++ b/benchmarks/utils.js @@ -0,0 +1,21 @@ +'use strict' + +function median (values) { + if (values.length === 0) { + return 0 + } + + values.sort((a, b) => a - b) + + var half = Math.floor(values.length / 2) + + if (values.length % 2) { + return values[half] + } + + return (values[half - 1] + values[half]) / 2.0 +} + +module.exports = { + median +} diff --git a/mysql-local/docker.js b/mysql-local/docker.js new file mode 100644 index 0000000..34878ea --- /dev/null +++ b/mysql-local/docker.js @@ -0,0 +1,33 @@ +'use strict' + +const delay = require('delay') +const execa = require('execa') +const pWaitFor = require('p-wait-for') + +module.exports = { + start: async (port = 3306, pw = 'test-secret-pw', database = 'libp2p_rendezvous_db') => { + const procResult = execa.commandSync(`docker run -p 3306:${port} -e MYSQL_ROOT_PASSWORD=${pw} -e MYSQL_DATABASE=${database} -d mysql:8 --default-authentication-plugin=mysql_native_password`, { + all: true + }) + const containerId = procResult.stdout + + console.log(`wait for docker container ${containerId} to be ready`) + + await pWaitFor(() => { + const procCheck = execa.commandSync(`docker logs ${containerId}`) + const logs = procCheck.stdout + procCheck.stderr // Docker/MySQL sends to the stderr the ready for connections... + + return logs.includes('ready for connections') + }, { + interval: 5000 + }) + // Some more time waiting to properly setup the container + await delay(12e3) + + return containerId + }, + stop: (containerId) => { + console.log('docker container is stopping') + execa.commandSync(`docker stop ${containerId}`) + } +} \ No newline at end of file diff --git a/package.json b/package.json index 7883b8f..3f2bdb6 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,6 @@ "peer-id": "^0.14.1", "protons": "^2.0.0", "set-delayed-interval": "^1.0.0", - "streaming-iterables": "^5.0.2", "uint8arrays": "^2.0.5" }, "devDependencies": { @@ -85,9 +84,12 @@ "execa": "^5.0.0", "ipfs-utils": "^5.0.1", "is-ci": "^2.0.0", + "microtime": "^3.0.0", "p-defer": "^3.0.0", "p-times": "^3.0.0", "p-wait-for": "^3.1.0", - "sinon": "^9.0.3" + "pidusage": "^2.0.21", + "sinon": "^9.0.3", + "streaming-iterables": "^5.0.3" } } diff --git a/src/bin.js b/src/bin.js index d18432b..d8d87e0 100644 --- a/src/bin.js +++ b/src/bin.js @@ -76,6 +76,7 @@ async function main () { } }, { datastore }) + console.log('Rendezvous server is starting') await rendezvousServer.start() console.log('Rendezvous server listening on:') rendezvousServer.multiaddrs.forEach((m) => console.log(m))