diff --git a/.gitignore b/.gitignore index c23d55e..d545580 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ node_modules .node_repl_history dist +test/example-repo/datastore/LOG* diff --git a/package.json b/package.json index 42dfc67..8979a49 100644 --- a/package.json +++ b/package.json @@ -32,14 +32,11 @@ "homepage": "https://github.com/ipld/js-ipld-resolver#readme", "license": "MIT", "devDependencies": { - "aegir": "^11.0.0", - "buffer-loader": "0.0.1", + "aegir": "^11.0.1", "chai": "^3.5.0", "dirty-chai": "^1.2.2", "eth-hash-to-cid": "^0.1.0", "ethereumjs-block": "^1.5.0", - "fs-pull-blob-store": "~0.4.1", - "idb-pull-blob-store": "~0.5.1", "lodash": "^4.17.4", "ncp": "^2.0.0", "pre-commit": "^1.2.2", @@ -49,20 +46,21 @@ "dependencies": { "async": "^2.1.5", "cids": "~0.4.2", - "interface-pull-blob-store": "~0.6.0", - "ipfs-block": "~0.5.5", - "ipfs-block-service": "~0.8.3", - "ipfs-repo": "~0.11.3", - "ipld-dag-cbor": "~0.10.1", - "ipld-dag-pb": "~0.10.1", - "ipld-eth-block": "^2.2.1", - "ipld-eth-block-list": "^1.0.3", - "ipld-eth-state-trie": "^1.0.2", - "ipld-eth-storage-trie": "^1.0.0", - "ipld-eth-tx-trie": "^1.0.0", + "interface-datastore": "^0.1.1", + "ipfs-block": "~0.6.0", + "ipfs-block-service": "~0.9.0", + "ipfs-repo": "~0.12.0", + "ipld-dag-cbor": "~0.11.0", + "ipld-dag-pb": "~0.11.0", + "ipld-eth-block": "^2.2.2", + "ipld-eth-block-list": "^1.0.4", + "ipld-eth-state-trie": "^1.0.4", + "ipld-eth-storage-trie": "^1.0.1", + "ipld-eth-tx-trie": "^1.0.1", "is-ipfs": "~0.3.0", "lodash.flatten": "^4.4.0", "lodash.includes": "^4.3.0", + "memdown": "^1.2.4", "multihashes": "~0.4.4", "pull-sort": "^1.0.0", "pull-stream": "^3.5.0", diff --git a/src/index.js b/src/index.js index 8c7053f..7207dc7 100644 --- a/src/index.js +++ b/src/index.js @@ -5,13 +5,14 @@ const pull = require('pull-stream') const CID = require('cids') const doUntil = require('async/doUntil') const IPFSRepo = require('ipfs-repo') -const MemoryStore = require('interface-pull-blob-store') const BlockService = require('ipfs-block-service') const joinPath = require('path').join const pullDeferSource = require('pull-defer').source const pullTraverse = require('pull-traverse') const map = require('async/map') +const series = require('async/series') const waterfall = require('async/waterfall') +const MemoryStore = require('interface-datastore').MemoryDatastore const dagPB = require('ipld-dag-pb') const dagCBOR = require('ipld-dag-cbor') @@ -25,10 +26,8 @@ function noop () {} class IPLDResolver { constructor (blockService) { - // nicola will love this! if (!blockService) { - const repo = new IPFSRepo('in-memory', { stores: MemoryStore }) - blockService = new BlockService(repo) + throw new Error('Missing blockservice') } this.bs = blockService @@ -182,50 +181,24 @@ class IPLDResolver { if (typeof options === 'function') { return setImmediate(() => callback(new Error('no options were passed'))) } - - let nodeAndCID + callback = callback || noop if (options.cid && CID.isCID(options.cid)) { - nodeAndCID = { - node: node, - cid: options.cid - } - - store.apply(this) - } else { - options.hashAlg = options.hashAlg || 'sha2-256' - - const r = this.resolvers[options.format] - // TODO add support for different hash funcs in the utils of - // each format (just really needed for CBOR for now, really - // r.util.cid(node1, hashAlg, (err, cid) => { - r.util.cid(node, (err, cid) => { - if (err) { - return callback(err) - } - - nodeAndCID = { - node: node, - cid: cid - } - - store.apply(this) - }) + return this._put(options.cid, node, callback) } - function store () { - callback = callback || noop + options.hashAlg = options.hashAlg || 'sha2-256' + const r = this.resolvers[options.format] + // TODO add support for different hash funcs in the utils of + // each format (just really needed for CBOR for now, really + // r.util.cid(node1, hashAlg, (err, cid) => { + r.util.cid(node, (err, cid) => { + if (err) { + return callback(err) + } - pull( - pull.values([nodeAndCID]), - this._putStream((err) => { - if (err) { - return callback(err) - } - callback(null, nodeAndCID.cid) - }) - ) - } + this._put(cid, node, callback) + }) } treeStream (cid, path, options) { @@ -340,22 +313,11 @@ class IPLDResolver { /* */ _get (cid, callback) { - pull( - this._getStream(cid), - pull.collect((err, res) => { - if (err) { - return callback(err) - } - callback(null, res[0]) - }) - ) - } + const r = this.resolvers[cid.codec] - _getStream (cid) { - return pull( - this.bs.getStream(cid), - pull.asyncMap((block, cb) => { - const r = this.resolvers[cid.codec] + waterfall([ + (cb) => this.bs.get(cid, cb), + (block, cb) => { if (r) { r.util.deserialize(block.data, (err, deserialized) => { if (err) { @@ -366,32 +328,50 @@ class IPLDResolver { } else { // multicodec unknown, send back raw data cb(null, block.data) } - }) - ) + } + ], callback) } - _putStream (callback) { + _put (cid, node, callback) { callback = callback || noop - return pull( - pull.asyncMap((nodeAndCID, cb) => { - const cid = nodeAndCID.cid - const r = this.resolvers[cid.codec] - - r.util.serialize(nodeAndCID.node, (err, serialized) => { - if (err) { - return cb(err) - } - cb(null, { - block: new Block(serialized), - cid: cid - }) - }) - }), - this.bs.putStream(), - pull.onEnd(callback) - ) + const r = this.resolvers[cid.codec] + waterfall([ + (cb) => r.util.serialize(node, cb), + (buf, cb) => this.bs.put(new Block(buf, cid), cb) + ], (err) => { + if (err) { + return callback(err) + } + callback(null, cid) + }) } } +/** + * Create an IPLD resolver with an inmemory blockservice and + * repo. + * + * @param {function(Error, IPLDResolver)} callback + * @returns {void} + */ +IPLDResolver.inMemory = function (callback) { + const repo = new IPFSRepo('in-memory', { + fs: MemoryStore, + level: require('memdown'), + lock: 'memory' + }) + const blockService = new BlockService(repo) + + series([ + (cb) => repo.init({}, cb), + (cb) => repo.open(cb) + ], (err) => { + if (err) { + return callback(err) + } + callback(null, new IPLDResolver(blockService)) + }) +} + module.exports = IPLDResolver diff --git a/test/basics.js b/test/basics.js index dd0b85c..cf27433 100644 --- a/test/basics.js +++ b/test/basics.js @@ -18,8 +18,10 @@ module.exports = (repo) => { }) it('creates an in memory repo if no blockService is passed', () => { - const r = new IPLDResolver() - expect(r.bs).to.exist() + IPLDResolver.inMemory((err, r) => { + expect(err).to.not.exist() + expect(r.bs).to.exist() + }) }) it.skip('add support to a new format', () => {}) diff --git a/test/browser.js b/test/browser.js index 3fe94fc..eb8f507 100644 --- a/test/browser.js +++ b/test/browser.js @@ -3,12 +3,8 @@ 'use strict' -const eachSeries = require('async/eachSeries') -const Store = require('idb-pull-blob-store') -const _ = require('lodash') +const series = require('async/series') const IPFSRepo = require('ipfs-repo') -const pull = require('pull-stream') -const repoContext = require.context('buffer!./example-repo', true) const basePath = 'ipfs' + Math.random() @@ -21,35 +17,25 @@ idb.deleteDatabase(basePath) idb.deleteDatabase(basePath + '/blocks') describe('Browser', () => { - before((done) => { - const repoData = [] - repoContext.keys().forEach((key) => { - repoData.push({ - key: key.replace('./', ''), - value: repoContext(key) - }) - }) - - const mainBlob = new Store(basePath) - const blocksBlob = new Store(basePath + '/blocks') - - eachSeries(repoData, (file, cb) => { - if (_.startsWith(file.key, 'datastore/')) { - return cb() - } - - const blocks = _.startsWith(file.key, 'blocks/') - const blob = blocks ? blocksBlob : mainBlob - const key = blocks ? file.key.replace(/^blocks\//, '') : file.key + const repo = new IPFSRepo(basePath) - pull( - pull.values([file.value]), - blob.write(key, cb) - ) - }, done) + before((done) => { + series([ + (cb) => repo.init({}, cb), + (cb) => repo.open(cb) + ], done) }) - const repo = new IPFSRepo(basePath, { stores: Store }) + after((done) => { + series([ + (cb) => repo.close(cb), + (cb) => { + idb.deleteDatabase(basePath) + idb.deleteDatabase(basePath + '/blocks') + cb() + } + ], done) + }) require('./basics')(repo) require('./ipld-dag-pb')(repo) diff --git a/test/example-repo/blocks/12204a5a/12204a5a95586f52e25811cf214677160e64383755a8c5163ba3c053c3b65777ed16.data b/test/example-repo/blocks/2F/CIQEUWUVLBXVFYSYCHHSCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data similarity index 100% rename from test/example-repo/blocks/12204a5a/12204a5a95586f52e25811cf214677160e64383755a8c5163ba3c053c3b65777ed16.data rename to test/example-repo/blocks/2F/CIQEUWUVLBXVFYSYCHHSCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data diff --git a/test/example-repo/blocks/122052c6/122052c63c7775396b3f82c639977a7223c2d96a9f70b5fd8b1d513f8c5b69dcaed4.data b/test/example-repo/blocks/5V/CIQFFRR4O52TS2Z7QLDDTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data similarity index 100% rename from test/example-repo/blocks/122052c6/122052c63c7775396b3f82c639977a7223c2d96a9f70b5fd8b1d513f8c5b69dcaed4.data rename to test/example-repo/blocks/5V/CIQFFRR4O52TS2Z7QLDDTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data diff --git a/test/example-repo/blocks/12209d6c/12209d6c2be50f706953479ab9df2ce3edca90b68053c00b3004b7f0accbe1e8eedf.data b/test/example-repo/blocks/5X/CIQJ23BL4UHXA2KTI6NLTXZM4PW4VEFWQBJ4ACZQAS37BLGL4HUO5XY.data similarity index 100% rename from test/example-repo/blocks/12209d6c/12209d6c2be50f706953479ab9df2ce3edca90b68053c00b3004b7f0accbe1e8eedf.data rename to test/example-repo/blocks/5X/CIQJ23BL4UHXA2KTI6NLTXZM4PW4VEFWQBJ4ACZQAS37BLGL4HUO5XY.data diff --git a/test/example-repo/blocks/1220c0fc/1220c0fc6b49543d7bf04e83d2a5a7cbe72a83e80f9c7bca1abcaa42298a57a33ff5.data b/test/example-repo/blocks/75/CIQMB7DLJFKD267QJ2B5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data similarity index 100% rename from test/example-repo/blocks/1220c0fc/1220c0fc6b49543d7bf04e83d2a5a7cbe72a83e80f9c7bca1abcaa42298a57a33ff5.data rename to test/example-repo/blocks/75/CIQMB7DLJFKD267QJ2B5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data diff --git a/test/example-repo/blocks/1220a52c/1220a52c3602030cb912edfe4de97002fdadf9d45666c3be122a2efb5db93c1d5fa6.data b/test/example-repo/blocks/7J/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data similarity index 100% rename from test/example-repo/blocks/1220a52c/1220a52c3602030cb912edfe4de97002fdadf9d45666c3be122a2efb5db93c1d5fa6.data rename to test/example-repo/blocks/7J/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data diff --git a/test/example-repo/blocks/1220e6a0/1220e6a045864ff8569e43e4866c0af807def07b0db2f018808620eb30b980e94011.data b/test/example-repo/blocks/AE/CIQONICFQZH7QVU6IPSIM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data similarity index 100% rename from test/example-repo/blocks/1220e6a0/1220e6a045864ff8569e43e4866c0af807def07b0db2f018808620eb30b980e94011.data rename to test/example-repo/blocks/AE/CIQONICFQZH7QVU6IPSIM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data diff --git a/test/example-repo/blocks/12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07606bfdb812303d.data b/test/example-repo/blocks/AP/CIQHAKDLTL5GMIFGN5YVY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data similarity index 100% rename from test/example-repo/blocks/12207028/122070286b9afa6620a66f715c7020d68af3d10e1a497971629c07606bfdb812303d.data rename to test/example-repo/blocks/AP/CIQHAKDLTL5GMIFGN5YVY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data diff --git a/test/example-repo/blocks/122031e7/122031e7a41c15d03feb8cd793c3348ea3b310512d7767a9abfbd7a928a85e977173.data b/test/example-repo/blocks/C4/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data similarity index 100% rename from test/example-repo/blocks/122031e7/122031e7a41c15d03feb8cd793c3348ea3b310512d7767a9abfbd7a928a85e977173.data rename to test/example-repo/blocks/C4/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data diff --git a/test/example-repo/blocks/12203628/12203628a4a19525dd84bbbffe132ec0b0d3be3528fbbcc814feb7f2fbf71ca06162.data b/test/example-repo/blocks/CY/CIQDMKFEUGKSLXMEXO774EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data similarity index 100% rename from test/example-repo/blocks/12203628/12203628a4a19525dd84bbbffe132ec0b0d3be3528fbbcc814feb7f2fbf71ca06162.data rename to test/example-repo/blocks/CY/CIQDMKFEUGKSLXMEXO774EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data diff --git a/test/example-repo/blocks/1220e5f8/1220e5f87b2b69cb032267bbc72a598eb49e3772f8949dacec55d099e3e17ac851c3.data b/test/example-repo/blocks/DQ/CIQOL6D3FNU4WAZCM654OKSZR22J4N3S7CKJ3LHMKXIJTY7BPLEFDQY.data similarity index 100% rename from test/example-repo/blocks/1220e5f8/1220e5f87b2b69cb032267bbc72a598eb49e3772f8949dacec55d099e3e17ac851c3.data rename to test/example-repo/blocks/DQ/CIQOL6D3FNU4WAZCM654OKSZR22J4N3S7CKJ3LHMKXIJTY7BPLEFDQY.data diff --git a/test/example-repo/blocks/12208b87/12208b872ca4ee517608331696dd6b3e5cf3497a7845ee8f94456ccf4d1d2f6602b5.data b/test/example-repo/blocks/FN/CIQIXBZMUTXFC5QIGMLJNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data similarity index 100% rename from test/example-repo/blocks/12208b87/12208b872ca4ee517608331696dd6b3e5cf3497a7845ee8f94456ccf4d1d2f6602b5.data rename to test/example-repo/blocks/FN/CIQIXBZMUTXFC5QIGMLJNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data diff --git a/test/example-repo/blocks/1220503a/1220503aec3bbd533b39fd56f5148750aa787e820fe98dbf68131bf70e796693d373.data b/test/example-repo/blocks/G4/CIQFAOXMHO6VGOZZ7VLPKFEHKCVHQ7UCB7UY3P3ICMN7ODTZM2J5G4Y.data similarity index 100% rename from test/example-repo/blocks/1220503a/1220503aec3bbd533b39fd56f5148750aa787e820fe98dbf68131bf70e796693d373.data rename to test/example-repo/blocks/G4/CIQFAOXMHO6VGOZZ7VLPKFEHKCVHQ7UCB7UY3P3ICMN7ODTZM2J5G4Y.data diff --git a/test/example-repo/blocks/12207fb8/12207fb898b5d7be46d85feb75d894e16cfa9a7ae5533f8e997cdab2ebadd7506340.data b/test/example-repo/blocks/GQ/CIQH7OEYWXL34RWYL7VXLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data similarity index 100% rename from test/example-repo/blocks/12207fb8/12207fb898b5d7be46d85feb75d894e16cfa9a7ae5533f8e997cdab2ebadd7506340.data rename to test/example-repo/blocks/GQ/CIQH7OEYWXL34RWYL7VXLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data diff --git a/test/example-repo/blocks/122031d6/122031d6da265092f1b03fec969243fdcf095c1d219356cdf538ffce705a52d5738d.data b/test/example-repo/blocks/HD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data similarity index 100% rename from test/example-repo/blocks/122031d6/122031d6da265092f1b03fec969243fdcf095c1d219356cdf538ffce705a52d5738d.data rename to test/example-repo/blocks/HD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data diff --git a/test/example-repo/blocks/1220929a/1220929a303c39da8a0b67c09697462f687a00c638bcb580feae06452e0c1f20b4.data b/test/example-repo/blocks/IL/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data similarity index 100% rename from test/example-repo/blocks/1220929a/1220929a303c39da8a0b67c09697462f687a00c638bcb580feae06452e0c1f20b4.data rename to test/example-repo/blocks/IL/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data diff --git a/test/example-repo/blocks/12206006/1220600627d80642365010791de1d62481e88d2c710099738ccbd6964075730194ba.data b/test/example-repo/blocks/JO/CIQGABRH3ADEENSQCB4R3YOWESA6RDJMOEAJS44MZPLJMQDVOMAZJOQ.data similarity index 100% rename from test/example-repo/blocks/12206006/1220600627d80642365010791de1d62481e88d2c710099738ccbd6964075730194ba.data rename to test/example-repo/blocks/JO/CIQGABRH3ADEENSQCB4R3YOWESA6RDJMOEAJS44MZPLJMQDVOMAZJOQ.data diff --git a/test/example-repo/blocks/12207d5c/12207d5cd815f41ac0566dd956c8541dced761633aca6704ba01a9be55934d95e5f1.data b/test/example-repo/blocks/L4/CIQH2XGYCX2BVQCWNXMVNSCUDXHNOYLDHLFGOBF2AGU34VMTJWK6L4I.data similarity index 100% rename from test/example-repo/blocks/12207d5c/12207d5cd815f41ac0566dd956c8541dced761633aca6704ba01a9be55934d95e5f1.data rename to test/example-repo/blocks/L4/CIQH2XGYCX2BVQCWNXMVNSCUDXHNOYLDHLFGOBF2AGU34VMTJWK6L4I.data diff --git a/test/example-repo/blocks/122090c0/122090c07a7795c1193510a696d1fdfc0f1e4947cff8e422610996e609dbcb976598.data b/test/example-repo/blocks/LG/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data similarity index 100% rename from test/example-repo/blocks/122090c0/122090c07a7795c1193510a696d1fdfc0f1e4947cff8e422610996e609dbcb976598.data rename to test/example-repo/blocks/LG/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data diff --git a/test/example-repo/blocks/1220e586/1220e586199640e1a4c63fa38c5434b9e72dc99d23a391d3db5e1e42d00527241671.data b/test/example-repo/blocks/M4/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data similarity index 100% rename from test/example-repo/blocks/1220e586/1220e586199640e1a4c63fa38c5434b9e72dc99d23a391d3db5e1e42d00527241671.data rename to test/example-repo/blocks/M4/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data diff --git a/test/example-repo/blocks/1220ec5b/1220ec5b533a3218991f4377b8b8c2538b95dd29d31eac6433af0fb6fcd83dd80778.data b/test/example-repo/blocks/O6/CIQOYW2THIZBRGI7IN33ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data similarity index 100% rename from test/example-repo/blocks/1220ec5b/1220ec5b533a3218991f4377b8b8c2538b95dd29d31eac6433af0fb6fcd83dd80778.data rename to test/example-repo/blocks/O6/CIQOYW2THIZBRGI7IN33ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data diff --git a/test/example-repo/blocks/1220d15b/1220d15b2cf7ecc6eccbf7e68c2e4290e8710bd97eafcce5380a127dcc5ea32a887f.data b/test/example-repo/blocks/Q7/CIQNCWZM67WMN3GL67TIYLSCSDUHCC6ZP2X4ZZJYBIJH3TC6UMVIQ7Y.data similarity index 100% rename from test/example-repo/blocks/1220d15b/1220d15b2cf7ecc6eccbf7e68c2e4290e8710bd97eafcce5380a127dcc5ea32a887f.data rename to test/example-repo/blocks/Q7/CIQNCWZM67WMN3GL67TIYLSCSDUHCC6ZP2X4ZZJYBIJH3TC6UMVIQ7Y.data diff --git a/test/example-repo/blocks/12206781/122067817186b8ff365c758f387e3ae7f28fa9367ee167c312e6d65a2e02e81ab815.data b/test/example-repo/blocks/QF/CIQGPALRQ24P6NS4OWHTQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data similarity index 100% rename from test/example-repo/blocks/12206781/122067817186b8ff365c758f387e3ae7f28fa9367ee167c312e6d65a2e02e81ab815.data rename to test/example-repo/blocks/QF/CIQGPALRQ24P6NS4OWHTQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data diff --git a/test/example-repo/blocks/1220e3b0/1220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.data b/test/example-repo/blocks/QV/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data similarity index 100% rename from test/example-repo/blocks/1220e3b0/1220e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.data rename to test/example-repo/blocks/QV/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data diff --git a/test/example-repo/blocks/1220120f/1220120f6af601d46e10b2d2e11ed71c55d25f3042c22501e41d1246e7a1e9d3d8ec.data b/test/example-repo/blocks/R3/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data similarity index 100% rename from test/example-repo/blocks/1220120f/1220120f6af601d46e10b2d2e11ed71c55d25f3042c22501e41d1246e7a1e9d3d8ec.data rename to test/example-repo/blocks/R3/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data diff --git a/test/example-repo/blocks/1220709b/1220709b2dcc5f6a90ad64d6fe3a5d202a72b057d1c7f2e682d0776a5363e2cca974.data b/test/example-repo/blocks/S5/CIQHBGZNZRPWVEFNMTLP4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data similarity index 100% rename from test/example-repo/blocks/1220709b/1220709b2dcc5f6a90ad64d6fe3a5d202a72b057d1c7f2e682d0776a5363e2cca974.data rename to test/example-repo/blocks/S5/CIQHBGZNZRPWVEFNMTLP4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data diff --git a/test/example-repo/blocks/SHARDING b/test/example-repo/blocks/SHARDING new file mode 100644 index 0000000..a153331 --- /dev/null +++ b/test/example-repo/blocks/SHARDING @@ -0,0 +1 @@ +/repo/flatfs/shard/v1/next-to-last/2 diff --git a/test/example-repo/blocks/12205200/12205200cc6b6f79e1588480d9f9016ccadfda3d8bc7d2f8cdf302aa51dae8dae9da.data b/test/example-repo/blocks/TW/CIQFEAGMNNXXTYKYQSANT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data similarity index 100% rename from test/example-repo/blocks/12205200/12205200cc6b6f79e1588480d9f9016ccadfda3d8bc7d2f8cdf302aa51dae8dae9da.data rename to test/example-repo/blocks/TW/CIQFEAGMNNXXTYKYQSANT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data diff --git a/test/example-repo/blocks/12205dff/12205dffb3c88f3de7036dc1840f0086f9fccbf5fa0106df39556b3b411ea4121a6f.data b/test/example-repo/blocks/U3/CIQF375TZCHT3ZYDNXAYIDYAQ347ZS7V7IAQNXZZKVVTWQI6UQJBU3Y.data similarity index 100% rename from test/example-repo/blocks/12205dff/12205dffb3c88f3de7036dc1840f0086f9fccbf5fa0106df39556b3b411ea4121a6f.data rename to test/example-repo/blocks/U3/CIQF375TZCHT3ZYDNXAYIDYAQ347ZS7V7IAQNXZZKVVTWQI6UQJBU3Y.data diff --git a/test/example-repo/blocks/1220e605/1220e605408ac3f78113ac9a7fd486441317afc9f967abe2aa05e7c641f7bbe98a37.data b/test/example-repo/blocks/UN/CIQOMBKARLB7PAITVSNH7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data similarity index 100% rename from test/example-repo/blocks/1220e605/1220e605408ac3f78113ac9a7fd486441317afc9f967abe2aa05e7c641f7bbe98a37.data rename to test/example-repo/blocks/UN/CIQOMBKARLB7PAITVSNH7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data diff --git a/test/example-repo/blocks/122062ce/122062ce1f2c91a13a97b596e873b5a3346a644605d7614dca53cd3a59f7385a8abb.data b/test/example-repo/blocks/VO/CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data similarity index 100% rename from test/example-repo/blocks/122062ce/122062ce1f2c91a13a97b596e873b5a3346a644605d7614dca53cd3a59f7385a8abb.data rename to test/example-repo/blocks/VO/CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data diff --git a/test/example-repo/blocks/12205994/122059948439065f29619ef41280cbb932be52c56d99c5966b65e0111239f098bbef.data b/test/example-repo/blocks/X3/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data similarity index 100% rename from test/example-repo/blocks/12205994/122059948439065f29619ef41280cbb932be52c56d99c5966b65e0111239f098bbef.data rename to test/example-repo/blocks/X3/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data diff --git a/test/example-repo/blocks/1220933b/1220933b41d37fd4508cdff45930dff56baef91c7dc345e73d049ab570abe10dfbb9.data b/test/example-repo/blocks/XO/CIQJGO2B2N75IUEM372FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data similarity index 100% rename from test/example-repo/blocks/1220933b/1220933b41d37fd4508cdff45930dff56baef91c7dc345e73d049ab570abe10dfbb9.data rename to test/example-repo/blocks/XO/CIQJGO2B2N75IUEM372FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data diff --git a/test/example-repo/blocks/_README b/test/example-repo/blocks/_README new file mode 100644 index 0000000..23cb090 --- /dev/null +++ b/test/example-repo/blocks/_README @@ -0,0 +1,30 @@ +This is a repository of IPLD objects. Each IPLD object is in a single file, +named .data. Where is the +"base32" encoding of the CID (as specified in +https://github.com/multiformats/multibase) without the 'B' prefix. +All the object files are placed in a tree of directories, based on a +function of the CID. This is a form of sharding similar to +the objects directory in git repositories. Previously, we used +prefixes, we now use the next-to-last two charters. + + func NextToLast(base32cid string) { + nextToLastLen := 2 + offset := len(base32cid) - nextToLastLen - 1 + return str[offset : offset+nextToLastLen] + } + +For example, an object with a base58 CIDv1 of + + zb2rhYSxw4ZjuzgCnWSt19Q94ERaeFhu9uSqRgjSdx9bsgM6f + +has a base32 CIDv1 of + + BAFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA + +and will be placed at + + SC/AFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA.data + +with 'SC' being the last-to-next two characters and the 'B' at the +beginning of the CIDv1 string is the multibase prefix that is not +stored in the filename. diff --git a/test/example-repo/datastore/CURRENT b/test/example-repo/datastore/CURRENT index 875cf23..6ba31a3 100644 --- a/test/example-repo/datastore/CURRENT +++ b/test/example-repo/datastore/CURRENT @@ -1 +1 @@ -MANIFEST-000007 +MANIFEST-000009 diff --git a/test/example-repo/datastore/LOG b/test/example-repo/datastore/LOG deleted file mode 100644 index 863b68f..0000000 --- a/test/example-repo/datastore/LOG +++ /dev/null @@ -1,10 +0,0 @@ -=============== Dec 10, 2015 (PST) =============== -07:50:02.056578 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed -07:50:02.057231 db@open opening -07:50:02.057312 journal@recovery F·1 -07:50:02.057514 journal@recovery recovering @3 -07:50:02.058921 mem@flush created L0@5 N·4 S·1KiB "/ip..\xf6\xe4\xa9,v5":"/pk..\xf6\xe4\xa9,v6" -07:50:02.059983 db@janitor F·4 G·0 -07:50:02.060001 db@open done T·2.755926ms -07:50:02.073183 db@close closing -07:50:02.073285 db@close done T·97.522µs diff --git a/test/example-repo/datastore/LOG.old b/test/example-repo/datastore/LOG.old deleted file mode 100644 index 708351e..0000000 --- a/test/example-repo/datastore/LOG.old +++ /dev/null @@ -1,10 +0,0 @@ -=============== Dec 10, 2015 (PST) =============== -07:49:57.048841 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed -07:49:57.049014 db@open opening -07:49:57.049066 journal@recovery F·1 -07:49:57.049233 journal@recovery recovering @1 -07:49:57.049693 mem@flush created L0@2 N·2 S·211B "/lo..oot,v2":"/lo..ins,v1" -07:49:57.050381 db@janitor F·3 G·0 -07:49:57.050397 db@open done T·1.375431ms -07:49:57.064580 db@close closing -07:49:57.064655 db@close done T·72.59µs diff --git a/test/example-repo/datastore/MANIFEST-000007 b/test/example-repo/datastore/MANIFEST-000007 deleted file mode 100644 index 6af3b54..0000000 Binary files a/test/example-repo/datastore/MANIFEST-000007 and /dev/null differ diff --git a/test/example-repo/datastore/MANIFEST-000009 b/test/example-repo/datastore/MANIFEST-000009 new file mode 100644 index 0000000..a69b18a Binary files /dev/null and b/test/example-repo/datastore/MANIFEST-000009 differ diff --git a/test/example-repo/version b/test/example-repo/version index 00750ed..7ed6ff8 100644 --- a/test/example-repo/version +++ b/test/example-repo/version @@ -1 +1 @@ -3 +5 diff --git a/test/ipld-all.js b/test/ipld-all.js index 8779b3e..6758fab 100644 --- a/test/ipld-all.js +++ b/test/ipld-all.js @@ -13,8 +13,8 @@ const expect = chai.expect chai.use(dirtyChai) const dagPB = require('ipld-dag-pb') const dagCBOR = require('ipld-dag-cbor') -const series = require('async/series') -const pull = require('pull-stream') +const each = require('async/each') +const waterfall = require('async/waterfall') const IPLDResolver = require('../src') @@ -27,47 +27,36 @@ describe('IPLD Resolver for dag-cbor + dag-pb', () => { let cidPb before((done) => { - resolver = new IPLDResolver() - - series([ - (cb) => { - dagPB.DAGNode.create(new Buffer('I am inside a Protobuf'), (err, node) => { - expect(err).to.not.exist() - nodePb = node - cb() - }) + waterfall([ + (cb) => IPLDResolver.inMemory(cb), + (res, cb) => { + resolver = res + dagPB.DAGNode.create(new Buffer('I am inside a Protobuf'), cb) }, - (cb) => { - dagPB.util.cid(nodePb, (err, cid) => { - expect(err).to.not.exist() - cidPb = cid - cb() - }) + (node, cb) => { + nodePb = node + dagPB.util.cid(nodePb, cb) }, - (cb) => { + (cid, cb) => { + cidPb = cid nodeCbor = { someData: 'I am inside a Cbor object', pb: { '/': cidPb.toBaseEncodedString() } } - dagCBOR.util.cid(nodeCbor, (err, cid) => { - expect(err).to.not.exist() - cidCbor = cid - cb() - }) - } - ], store) + dagCBOR.util.cid(nodeCbor, cb) + }, + (cid, cb) => { + cidCbor = cid - function store () { - pull( - pull.values([ + each([ { node: nodePb, cid: cidPb }, { node: nodeCbor, cid: cidCbor } - ]), - pull.asyncMap((nac, cb) => resolver.put(nac.node, { cid: nac.cid }, cb)), - pull.onEnd(done) - ) - } + ], (nac, cb) => { + resolver.put(nac.node, { cid: nac.cid }, cb) + }, cb) + } + ], done) }) it('resolve through different formats', (done) => { diff --git a/test/ipld-dag-cbor.js b/test/ipld-dag-cbor.js index f3a4b0b..4b488fd 100644 --- a/test/ipld-dag-cbor.js +++ b/test/ipld-dag-cbor.js @@ -8,6 +8,7 @@ chai.use(dirtyChai) const BlockService = require('ipfs-block-service') const dagCBOR = require('ipld-dag-cbor') const series = require('async/series') +const each = require('async/each') const pull = require('pull-stream') const IPLDResolver = require('../src') @@ -79,15 +80,14 @@ module.exports = (repo) => { }) describe('internals', () => { - it('resolver._putStream', (done) => { - pull( - pull.values([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ]), - resolver._putStream(done) - ) + it('resolver._put', (done) => { + each([ + { node: node1, cid: cid1 }, + { node: node2, cid: cid2 }, + { node: node3, cid: cid3 } + ], (nc, cb) => { + resolver._put(nc.cid, nc.node, cb) + }, done) }) it('resolver._get', (done) => { @@ -100,20 +100,6 @@ module.exports = (repo) => { }) }) }) - - it('resolver._getStream', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - pull( - resolver._getStream(cid1), - pull.collect((err, nodes) => { - expect(err).to.not.exist() - expect(node1).to.eql(nodes[0]) - done() - }) - ) - }) - }) }) describe('public api', () => { diff --git a/test/ipld-dag-pb.js b/test/ipld-dag-pb.js index 0af3a79..d7b66bd 100644 --- a/test/ipld-dag-pb.js +++ b/test/ipld-dag-pb.js @@ -8,6 +8,7 @@ chai.use(dirtyChai) const BlockService = require('ipfs-block-service') const dagPB = require('ipld-dag-pb') const series = require('async/series') +const each = require('async/each') const pull = require('pull-stream') const IPLDResolver = require('../src') @@ -124,40 +125,23 @@ module.exports = (repo) => { }) describe('internals', () => { - it('resolver._putStream', (done) => { - pull( - pull.values([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ]), - resolver._putStream(done) - ) - }) - - it('resolver._getStream', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - pull( - resolver._getStream(cid1), - pull.collect((err, nodes) => { - expect(err).to.not.exist() - done() - }) - ) - }) + it('resolver._put', (done) => { + each([ + { node: node1, cid: cid1 }, + { node: node2, cid: cid2 }, + { node: node3, cid: cid3 } + ], (nc, cb) => { + resolver._put(nc.cid, nc.node, cb) + }, done) }) it('resolver._get', (done) => { resolver.put(node1, { cid: cid1 }, (err) => { expect(err).to.not.exist() - pull( - resolver._getStream(cid1), - pull.collect((err, nodes) => { - expect(err).to.not.exist() - done() - }) - ) + resolver._get(cid1, (err, node) => { + expect(err).to.not.exist() + done() + }) }) }) }) diff --git a/test/ipld-eth-block.js b/test/ipld-eth-block.js index 9cb88e2..35ebe45 100644 --- a/test/ipld-eth-block.js +++ b/test/ipld-eth-block.js @@ -9,6 +9,7 @@ const BlockService = require('ipfs-block-service') const ipldEthBlock = require('ipld-eth-block') const EthBlockHeader = require('ethereumjs-block/header') const series = require('async/series') +const each = require('async/each') const pull = require('pull-stream') const IPLDResolver = require('../src') @@ -80,15 +81,14 @@ module.exports = (repo) => { }) describe('internals', () => { - it('resolver._putStream', (done) => { - pull( - pull.values([ - { node: node1, cid: cid1 }, - { node: node2, cid: cid2 }, - { node: node3, cid: cid3 } - ]), - resolver._putStream(done) - ) + it('resolver._put', (done) => { + each([ + { node: node1, cid: cid1 }, + { node: node2, cid: cid2 }, + { node: node3, cid: cid3 } + ], (nc, cb) => { + resolver._put(nc.cid, nc.node, cb) + }, done) }) it('resolver._get', (done) => { @@ -103,22 +103,6 @@ module.exports = (repo) => { }) }) }) - - it('resolver._getStream', (done) => { - resolver.put(node1, { cid: cid1 }, (err) => { - expect(err).to.not.exist() - pull( - resolver._getStream(cid1), - pull.collect((err, results) => { - expect(err).to.not.exist() - const node = results[0] - expect(node1.raw).to.eql(node.raw) - expect(node1.hash()).to.eql(node.hash()) - done() - }) - ) - }) - }) }) describe('public api', () => { diff --git a/test/node.js b/test/node.js index ca69c6f..21a74ae 100644 --- a/test/node.js +++ b/test/node.js @@ -4,17 +4,27 @@ const ncp = require('ncp').ncp const rimraf = require('rimraf') const IPFSRepo = require('ipfs-repo') -const Store = require('fs-pull-blob-store') +const series = require('async/series') const os = require('os') describe('Node.js', () => { const repoExample = process.cwd() + '/test/example-repo' const repoTests = os.tmpDir() + '/t-r-' + Date.now() + const repo = new IPFSRepo(repoTests) - before((done) => ncp(repoExample, repoTests, done)) - after((done) => rimraf(repoTests, done)) + before((done) => { + series([ + (cb) => ncp(repoExample, repoTests, cb), + (cb) => repo.open(cb) + ], done) + }) - const repo = new IPFSRepo(repoTests, { stores: Store }) + after((done) => { + series([ + (cb) => repo.close(cb), + (cb) => rimraf(repoTests, cb) + ], done) + }) require('./basics')(repo) require('./ipld-dag-pb')(repo)