This repository was archived by the owner on Oct 1, 2021. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: migration 10 to allow upgrading level in the browser
We use the [level](https://www.npmjs.com/package/level) module to supply either [leveldown](http://npmjs.com/package/leveldown) or [level-js](https://www.npmjs.com/package/level-js) to [datastore-level](https://www.npmjs.com/package/datastore-level) depending on if we're running under node or in the browser. `level@6.x.x` upgrades the `level-js` dependency from `4.x.x` to `5.x.x` which includes the changes from [Level/level-js#179](Level/level-js#179) so `>5.x.x` requires all database keys/values to be Uint8Arrays and they can no longer be strings. We already store values as Uint8Arrays but our keys are strings, so here we add a migration to converts all datastore keys to Uint8Arrays. N.b. `leveldown` already does this conversion for us so this migration only needs to run in the browser.
- Loading branch information
1 parent
d0866b1
commit cb7167c
Showing
8 changed files
with
336 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,168 @@ | ||
| 'use strict' | ||
|
|
||
| const { createStore } = require('../../src/utils') | ||
| const { Key } = require('interface-datastore') | ||
| const fromString = require('uint8arrays/from-string') | ||
| const toString = require('uint8arrays/to-string') | ||
|
|
||
| const findUpgradableDb = (store) => { | ||
| let db = store | ||
|
|
||
| while (db.db || db.child) { | ||
| db = db.db || db.child | ||
|
|
||
| // Will stop at Level in the browser, LevelDOWN in node | ||
| if (db.constructor.name === 'Level') { | ||
| return db | ||
| } | ||
| } | ||
| } | ||
|
|
||
| async function keysToBinary (name, store, onProgress = () => {}) { | ||
| let db = findUpgradableDb(store) | ||
|
|
||
| // only interested in Level | ||
| if (!db) { | ||
| onProgress(`${name} did not need an upgrade`) | ||
|
|
||
| return | ||
| } | ||
|
|
||
| onProgress(`Upgrading ${name}`) | ||
|
|
||
| await withEach(db, (key, value) => { | ||
| return [ | ||
| { type: 'del', key: key }, | ||
| { type: 'put', key: fromString(key), value: value } | ||
| ] | ||
| }) | ||
| } | ||
|
|
||
| async function keysToStrings (name, store, onProgress = () => {}) { | ||
| let db = findUpgradableDb(store) | ||
|
|
||
| // only interested in Level | ||
| if (!db) { | ||
| onProgress(`${name} did not need a downgrade`) | ||
|
|
||
| return | ||
| } | ||
|
|
||
| onProgress(`Downgrading ${name}`) | ||
|
|
||
| await withEach(db, (key, value) => { | ||
| return [ | ||
| { type: 'del', key: key }, | ||
| { type: 'put', key: toString(key), value: value } | ||
| ] | ||
| }) | ||
| } | ||
|
|
||
| async function process (repoPath, repoOptions, onProgress, fn) { | ||
| const datastores = Object.keys(repoOptions.storageBackends) | ||
| .filter(key => repoOptions.storageBackends[key].name === 'LevelDatastore') | ||
| .map(name => ({ | ||
| name, | ||
| store: createStore(repoPath, name, repoOptions) | ||
| })) | ||
|
|
||
| onProgress(0, `Migrating ${datastores.length} dbs`) | ||
| let migrated = 0 | ||
|
|
||
| for (const { name, store } of datastores) { | ||
| await store.open() | ||
|
|
||
| try { | ||
| await fn(name, store, (message) => { | ||
| onProgress(parseInt((migrated / datastores.length) * 100), message) | ||
| }) | ||
| } finally { | ||
| migrated++ | ||
| store.close() | ||
| } | ||
| } | ||
|
|
||
| onProgress(100, `Migrated ${datastores.length} dbs`) | ||
| } | ||
|
|
||
| module.exports = { | ||
| version: 10, | ||
| description: 'Migrates datastore-level keys to binary', | ||
| migrate: (repoPath, repoOptions, onProgress) => { | ||
| return process(repoPath, repoOptions, onProgress, keysToBinary) | ||
| }, | ||
| revert: (repoPath, repoOptions, onProgress) => { | ||
| return process(repoPath, repoOptions, onProgress, keysToStrings) | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * @typedef {Error | undefined} Err | ||
| * @typedef {Uint8Array|string} Key | ||
| * @typedef {Uint8Array} Value | ||
| * @typedef {{ type: 'del', key: Key } | { type: 'put', key: Key, value: Value }} Operation | ||
| * | ||
| * Uses the upgrade strategy from level-js@5.x.x - note we can't call the `.upgrade` command | ||
| * directly because it will be removed in level-js@6.x.x and we can't guarantee users will | ||
| * have migrated by then - e.g. they may jump from level-js@4.x.x straight to level-js@6.x.x | ||
| * so we have to duplicate the code here. | ||
| * | ||
| * @param {import('interface-datastore').Datastore} db | ||
| * @param {function (Err, Key, Value): Operation[]} fn | ||
| */ | ||
| function withEach (db, fn) { | ||
| function batch (operations, next) { | ||
| const store = db.store('readwrite') | ||
| const transaction = store.transaction | ||
| let index = 0 | ||
| let error | ||
|
|
||
| transaction.onabort = () => next(error || transaction.error || new Error('aborted by user')) | ||
| transaction.oncomplete = () => next() | ||
|
|
||
| function loop () { | ||
| var op = operations[index++] | ||
| var key = op.key | ||
|
|
||
| try { | ||
| var req = op.type === 'del' ? store.delete(key) : store.put(op.value, key) | ||
| } catch (err) { | ||
| error = err | ||
| transaction.abort() | ||
| return | ||
| } | ||
|
|
||
| if (index < operations.length) { | ||
| req.onsuccess = loop | ||
| } | ||
| } | ||
|
|
||
| loop() | ||
| } | ||
|
|
||
| return new Promise((resolve, reject) => { | ||
| const it = db.iterator() | ||
| // raw keys and values only | ||
| it._deserializeKey = it._deserializeValue = (data) => data | ||
| next() | ||
|
|
||
| function next () { | ||
| it.next((err, key, value) => { | ||
| if (err || key === undefined) { | ||
| it.end((err2) => { | ||
| if (err2) { | ||
| reject(err2) | ||
| return | ||
| } | ||
|
|
||
| resolve() | ||
| }) | ||
|
|
||
| return | ||
| } | ||
|
|
||
| batch(fn(key, value), next) | ||
| }) | ||
| } | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,125 @@ | ||
| /* eslint-env mocha */ | ||
| /* eslint-disable max-nested-callbacks */ | ||
| 'use strict' | ||
|
|
||
| const { expect } = require('aegir/utils/chai') | ||
|
|
||
| const { createStore } = require('../../src/utils') | ||
| const migration = require('../../migrations/migration-10') | ||
| const Key = require('interface-datastore').Key | ||
| const fromString = require('uint8arrays/from-string') | ||
| const Level5 = require('level-5') | ||
| const Level6 = require('level-6') | ||
|
|
||
| const keys = { | ||
| CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I: 'hello', | ||
| CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ: fromString('derp') | ||
| } | ||
|
|
||
| async function bootstrap (dir, backend, repoOptions) { | ||
| const store = createStore(dir, backend, repoOptions) | ||
| await store.open() | ||
|
|
||
| for (const name of Object.keys(keys)) { | ||
| await store.put(new Key(name), keys[name]) | ||
| } | ||
|
|
||
| await store.close() | ||
| } | ||
|
|
||
| async function validate (dir, backend, repoOptions) { | ||
| const store = createStore(dir, backend, repoOptions) | ||
|
|
||
| await store.open() | ||
|
|
||
| for (const name of Object.keys(keys)) { | ||
| const key = new Key(`/${name}`) | ||
|
|
||
| expect(await store.has(key)).to.be.true(`Could not read key ${name}`) | ||
| expect(store.get(key)).to.eventually.equal(keys[name], `Could not read value for key ${keys[name]}`) | ||
| } | ||
|
|
||
| await store.close() | ||
| } | ||
|
|
||
| function withLevel (repoOptions, levelImpl) { | ||
| const stores = Object.keys(repoOptions.storageBackends) | ||
| .filter(key => repoOptions.storageBackends[key].name === 'LevelDatastore') | ||
|
|
||
| const output = { | ||
| ...repoOptions | ||
| } | ||
|
|
||
| stores.forEach(store => { | ||
| // override version of level passed to datastore options | ||
| output.storageBackendOptions[store] = { | ||
| ...output.storageBackendOptions[store], | ||
| db: levelImpl | ||
| } | ||
| }) | ||
|
|
||
| return output | ||
| } | ||
|
|
||
| module.exports = (setup, cleanup, repoOptions) => { | ||
| describe('migration 10', function () { | ||
| this.timeout(240 * 1000) | ||
| let dir | ||
|
|
||
| beforeEach(async () => { | ||
| dir = await setup() | ||
| }) | ||
|
|
||
| afterEach(async () => { | ||
| await cleanup(dir) | ||
| }) | ||
|
|
||
| describe('forwards', () => { | ||
| beforeEach(async () => { | ||
| for (const backend of Object.keys(repoOptions.storageBackends)) { | ||
| await bootstrap(dir, backend, withLevel(repoOptions, Level5)) | ||
| } | ||
| }) | ||
|
|
||
| it('should migrate keys and values forward', async () => { | ||
| await migration.migrate(dir, withLevel(repoOptions, Level6), () => {}) | ||
|
|
||
| for (const backend of Object.keys(repoOptions.storageBackends)) { | ||
| await validate(dir, backend, withLevel(repoOptions, Level6)) | ||
| } | ||
| }) | ||
| }) | ||
|
|
||
| describe('backwards using level@6.x.x', () => { | ||
| beforeEach(async () => { | ||
| for (const backend of Object.keys(repoOptions.storageBackends)) { | ||
| await bootstrap(dir, backend, withLevel(repoOptions, Level6)) | ||
| } | ||
| }) | ||
|
|
||
| it('should migrate keys and values backward', async () => { | ||
| await migration.revert(dir, withLevel(repoOptions, Level6), () => {}) | ||
|
|
||
| for (const backend of Object.keys(repoOptions.storageBackends)) { | ||
| await validate(dir, backend, withLevel(repoOptions, Level5)) | ||
| } | ||
| }) | ||
| }) | ||
|
|
||
| describe('backwards using level@5.x.x', () => { | ||
| beforeEach(async () => { | ||
| for (const backend of Object.keys(repoOptions.storageBackends)) { | ||
| await bootstrap(dir, backend, withLevel(repoOptions, Level6)) | ||
| } | ||
| }) | ||
|
|
||
| it('should migrate keys and values backward', async () => { | ||
| await migration.revert(dir, withLevel(repoOptions, Level5), () => {}) | ||
|
|
||
| for (const backend of Object.keys(repoOptions.storageBackends)) { | ||
| await validate(dir, backend, withLevel(repoOptions, Level5)) | ||
| } | ||
| }) | ||
| }) | ||
| }) | ||
| } |
Oops, something went wrong.