From cb7167c53570e88cbf9517a1f5546b0d0ec1c741 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 28 Jan 2021 12:41:42 +0000 Subject: [PATCH] feat: migration 10 to allow upgrading level in the browser We use the [level](https://www.npmjs.com/package/level) module to supply either [leveldown](http://npmjs.com/package/leveldown) or [level-js](https://www.npmjs.com/package/level-js) to [datastore-level](https://www.npmjs.com/package/datastore-level) depending on if we're running under node or in the browser. `level@6.x.x` upgrades the `level-js` dependency from `4.x.x` to `5.x.x` which includes the changes from [Level/level-js#179](https://github.com/Level/level-js/pull/179) so `>5.x.x` requires all database keys/values to be Uint8Arrays and they can no longer be strings. We already store values as Uint8Arrays but our keys are strings, so here we add a migration to converts all datastore keys to Uint8Arrays. N.b. `leveldown` already does this conversion for us so this migration only needs to run in the browser. --- README.md | 23 ++++ migrations/index.js | 3 +- migrations/migration-10/index.js | 168 +++++++++++++++++++++++++++ package.json | 2 + test/browser.js | 13 ++- test/migrations/index.js | 1 + test/migrations/migration-10-test.js | 125 ++++++++++++++++++++ test/node.js | 14 ++- 8 files changed, 336 insertions(+), 13 deletions(-) create mode 100644 migrations/migration-10/index.js create mode 100644 test/migrations/migration-10-test.js diff --git a/README.md b/README.md index a5d28e3..d22f2d1 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,11 @@ This package is inspired by the [go-ipfs repo migration tool](https://github.com - [Tests](#tests) - [Empty migrations](#empty-migrations) - [Migrations matrix](#migrations-matrix) + - [Migrations](#migrations) + - [7](#7) + - [8](#8) + - [9](#9) + - [10](#10) - [Developer](#developer) - [Module versioning notes](#module-versioning-notes) - [Contribute](#contribute) @@ -268,6 +273,24 @@ This will create an empty migration with the next version. | 8 | v0.48.0 | | 9 | v0.49.0 | +### Migrations + +#### 7 + +This is the initial version of the datastore, inherited from go-IPFS in an attempt to maintain cross-compatibility between the two implementations. + +#### 8 + +Blockstore keys are transformed into base32 representations of the multihash from the CID of the block. + +#### 9 + +Pins were migrated from a DAG to a Datastore - see [ipfs/js-ipfs#2771](https://github.com/ipfs/js-ipfs/pull/2771) + +#### 10 + +`level@6.x.x` upgrades the `level-js` dependency from `4.x.x` to `5.x.x`. This update requires a database migration to convert all string keys/values into buffers. Only runs in the browser, node is unaffected. See [Level/level-js#179](https://github.com/Level/level-js/pull/179) + ## Developer ### Module versioning notes diff --git a/migrations/index.js b/migrations/index.js index 11ceced..91b83df 100644 --- a/migrations/index.js +++ b/migrations/index.js @@ -16,5 +16,6 @@ module.exports = [ Object.assign({version: 6}, emptyMigration), Object.assign({version: 7}, emptyMigration), require('./migration-8'), - require('./migration-9') + require('./migration-9'), + require('./migration-10') ] diff --git a/migrations/migration-10/index.js b/migrations/migration-10/index.js new file mode 100644 index 0000000..f7c4026 --- /dev/null +++ b/migrations/migration-10/index.js @@ -0,0 +1,168 @@ +'use strict' + +const { createStore } = require('../../src/utils') +const { Key } = require('interface-datastore') +const fromString = require('uint8arrays/from-string') +const toString = require('uint8arrays/to-string') + +const findUpgradableDb = (store) => { + let db = store + + while (db.db || db.child) { + db = db.db || db.child + + // Will stop at Level in the browser, LevelDOWN in node + if (db.constructor.name === 'Level') { + return db + } + } +} + +async function keysToBinary (name, store, onProgress = () => {}) { + let db = findUpgradableDb(store) + + // only interested in Level + if (!db) { + onProgress(`${name} did not need an upgrade`) + + return + } + + onProgress(`Upgrading ${name}`) + + await withEach(db, (key, value) => { + return [ + { type: 'del', key: key }, + { type: 'put', key: fromString(key), value: value } + ] + }) +} + +async function keysToStrings (name, store, onProgress = () => {}) { + let db = findUpgradableDb(store) + + // only interested in Level + if (!db) { + onProgress(`${name} did not need a downgrade`) + + return + } + + onProgress(`Downgrading ${name}`) + + await withEach(db, (key, value) => { + return [ + { type: 'del', key: key }, + { type: 'put', key: toString(key), value: value } + ] + }) +} + +async function process (repoPath, repoOptions, onProgress, fn) { + const datastores = Object.keys(repoOptions.storageBackends) + .filter(key => repoOptions.storageBackends[key].name === 'LevelDatastore') + .map(name => ({ + name, + store: createStore(repoPath, name, repoOptions) + })) + + onProgress(0, `Migrating ${datastores.length} dbs`) + let migrated = 0 + + for (const { name, store } of datastores) { + await store.open() + + try { + await fn(name, store, (message) => { + onProgress(parseInt((migrated / datastores.length) * 100), message) + }) + } finally { + migrated++ + store.close() + } + } + + onProgress(100, `Migrated ${datastores.length} dbs`) +} + +module.exports = { + version: 10, + description: 'Migrates datastore-level keys to binary', + migrate: (repoPath, repoOptions, onProgress) => { + return process(repoPath, repoOptions, onProgress, keysToBinary) + }, + revert: (repoPath, repoOptions, onProgress) => { + return process(repoPath, repoOptions, onProgress, keysToStrings) + } +} + +/** + * @typedef {Error | undefined} Err + * @typedef {Uint8Array|string} Key + * @typedef {Uint8Array} Value + * @typedef {{ type: 'del', key: Key } | { type: 'put', key: Key, value: Value }} Operation + * + * Uses the upgrade strategy from level-js@5.x.x - note we can't call the `.upgrade` command + * directly because it will be removed in level-js@6.x.x and we can't guarantee users will + * have migrated by then - e.g. they may jump from level-js@4.x.x straight to level-js@6.x.x + * so we have to duplicate the code here. + * + * @param {import('interface-datastore').Datastore} db + * @param {function (Err, Key, Value): Operation[]} fn + */ +function withEach (db, fn) { + function batch (operations, next) { + const store = db.store('readwrite') + const transaction = store.transaction + let index = 0 + let error + + transaction.onabort = () => next(error || transaction.error || new Error('aborted by user')) + transaction.oncomplete = () => next() + + function loop () { + var op = operations[index++] + var key = op.key + + try { + var req = op.type === 'del' ? store.delete(key) : store.put(op.value, key) + } catch (err) { + error = err + transaction.abort() + return + } + + if (index < operations.length) { + req.onsuccess = loop + } + } + + loop() + } + + return new Promise((resolve, reject) => { + const it = db.iterator() + // raw keys and values only + it._deserializeKey = it._deserializeValue = (data) => data + next() + + function next () { + it.next((err, key, value) => { + if (err || key === undefined) { + it.end((err2) => { + if (err2) { + reject(err2) + return + } + + resolve() + }) + + return + } + + batch(fn(key, value), next) + }) + } + }) +} diff --git a/package.json b/package.json index cde37fd..e1c87f8 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,8 @@ "datastore-level": "^3.0.0", "it-all": "^1.0.2", "just-safe-set": "^2.1.0", + "level-5": "npm:level@^5.0.0", + "level-6": "npm:level@^6.0.0", "ncp": "^2.0.0", "rimraf": "^3.0.0", "sinon": "^9.0.2" diff --git a/test/browser.js b/test/browser.js index 2fd1640..f9207e4 100644 --- a/test/browser.js +++ b/test/browser.js @@ -1,16 +1,17 @@ /* eslint-env mocha */ 'use strict' +const DatastoreLevel = require('datastore-level') const { createRepo, createAndLoadRepo } = require('./fixtures/repo') const repoOptions = { lock: 'memory', storageBackends: { - root: require('datastore-level'), - blocks: require('datastore-level'), - keys: require('datastore-level'), - datastore: require('datastore-level'), - pins: require('datastore-level') + root: DatastoreLevel, + blocks: DatastoreLevel, + keys: DatastoreLevel, + datastore: DatastoreLevel, + pins: DatastoreLevel }, storageBackendOptions: { root: { @@ -51,7 +52,7 @@ describe('Browser specific tests', () => { }) describe('migrations tests', () => { - require('./migrations')(() => createRepo(repoOptions), repoCleanup, repoOptions) + require('./migrations')(() => createRepo(repoOptions), repoCleanup) }) describe('init tests', () => { diff --git a/test/migrations/index.js b/test/migrations/index.js index 50c3016..bdc4129 100644 --- a/test/migrations/index.js +++ b/test/migrations/index.js @@ -77,6 +77,7 @@ module.exports = (createRepo, repoCleanup) => { describe(name, () => { require('./migration-8-test')(createRepo, repoCleanup, options) require('./migration-9-test')(createRepo, repoCleanup, options) + require('./migration-10-test')(createRepo, repoCleanup, options) }) }) } diff --git a/test/migrations/migration-10-test.js b/test/migrations/migration-10-test.js new file mode 100644 index 0000000..b3340fc --- /dev/null +++ b/test/migrations/migration-10-test.js @@ -0,0 +1,125 @@ +/* eslint-env mocha */ +/* eslint-disable max-nested-callbacks */ +'use strict' + +const { expect } = require('aegir/utils/chai') + +const { createStore } = require('../../src/utils') +const migration = require('../../migrations/migration-10') +const Key = require('interface-datastore').Key +const fromString = require('uint8arrays/from-string') +const Level5 = require('level-5') +const Level6 = require('level-6') + +const keys = { + CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I: 'hello', + CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ: fromString('derp') +} + +async function bootstrap (dir, backend, repoOptions) { + const store = createStore(dir, backend, repoOptions) + await store.open() + + for (const name of Object.keys(keys)) { + await store.put(new Key(name), keys[name]) + } + + await store.close() +} + +async function validate (dir, backend, repoOptions) { + const store = createStore(dir, backend, repoOptions) + + await store.open() + + for (const name of Object.keys(keys)) { + const key = new Key(`/${name}`) + + expect(await store.has(key)).to.be.true(`Could not read key ${name}`) + expect(store.get(key)).to.eventually.equal(keys[name], `Could not read value for key ${keys[name]}`) + } + + await store.close() +} + +function withLevel (repoOptions, levelImpl) { + const stores = Object.keys(repoOptions.storageBackends) + .filter(key => repoOptions.storageBackends[key].name === 'LevelDatastore') + + const output = { + ...repoOptions + } + + stores.forEach(store => { + // override version of level passed to datastore options + output.storageBackendOptions[store] = { + ...output.storageBackendOptions[store], + db: levelImpl + } + }) + + return output +} + +module.exports = (setup, cleanup, repoOptions) => { + describe('migration 10', function () { + this.timeout(240 * 1000) + let dir + + beforeEach(async () => { + dir = await setup() + }) + + afterEach(async () => { + await cleanup(dir) + }) + + describe('forwards', () => { + beforeEach(async () => { + for (const backend of Object.keys(repoOptions.storageBackends)) { + await bootstrap(dir, backend, withLevel(repoOptions, Level5)) + } + }) + + it('should migrate keys and values forward', async () => { + await migration.migrate(dir, withLevel(repoOptions, Level6), () => {}) + + for (const backend of Object.keys(repoOptions.storageBackends)) { + await validate(dir, backend, withLevel(repoOptions, Level6)) + } + }) + }) + + describe('backwards using level@6.x.x', () => { + beforeEach(async () => { + for (const backend of Object.keys(repoOptions.storageBackends)) { + await bootstrap(dir, backend, withLevel(repoOptions, Level6)) + } + }) + + it('should migrate keys and values backward', async () => { + await migration.revert(dir, withLevel(repoOptions, Level6), () => {}) + + for (const backend of Object.keys(repoOptions.storageBackends)) { + await validate(dir, backend, withLevel(repoOptions, Level5)) + } + }) + }) + + describe('backwards using level@5.x.x', () => { + beforeEach(async () => { + for (const backend of Object.keys(repoOptions.storageBackends)) { + await bootstrap(dir, backend, withLevel(repoOptions, Level6)) + } + }) + + it('should migrate keys and values backward', async () => { + await migration.revert(dir, withLevel(repoOptions, Level5), () => {}) + + for (const backend of Object.keys(repoOptions.storageBackends)) { + await validate(dir, backend, withLevel(repoOptions, Level5)) + } + }) + }) + }) +} diff --git a/test/node.js b/test/node.js index 7e97142..6c26abe 100644 --- a/test/node.js +++ b/test/node.js @@ -1,6 +1,8 @@ /* eslint-env mocha */ 'use strict' +const DatastoreFS = require('datastore-fs') +const DatastoreLevel = require('datastore-level') const promisify = require('util').promisify const asyncRimraf = promisify(require('rimraf')) const { createRepo, createAndLoadRepo } = require('./fixtures/repo') @@ -9,11 +11,11 @@ const os = require('os') const repoOptions = { lock: 'fs', storageBackends: { - root: require('datastore-fs'), - blocks: require('datastore-fs'), - keys: require('datastore-fs'), - datastore: require('datastore-level'), - pins: require('datastore-level') + root: DatastoreFS, + blocks: DatastoreFS, + keys: DatastoreFS, + datastore: DatastoreLevel, + pins: DatastoreLevel }, storageBackendOptions: { root: { @@ -50,7 +52,7 @@ describe('Node specific tests', () => { }) describe('migrations tests', () => { - require('./migrations')(() => createRepo(repoOptions, os.tmpdir()), repoCleanup, repoOptions) + require('./migrations')(() => createRepo(repoOptions, os.tmpdir()), repoCleanup) }) describe('init tests', () => {