From d102ad006fe01ba31bfe8d2748bafbd4340f4889 Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Sun, 18 Aug 2019 11:12:55 +0300 Subject: [PATCH] Add clear() method to delete all entries or a range (#310) --- README.md | 26 ++++ abstract-leveldown.js | 46 +++++++ test/clear-range-test.js | 258 +++++++++++++++++++++++++++++++++++++++ test/clear-test.js | 83 +++++++++++++ test/common.js | 23 +++- test/index.js | 5 + test/self.js | 111 +++++++++++++++-- 7 files changed, 539 insertions(+), 13 deletions(-) create mode 100644 test/clear-range-test.js create mode 100644 test/clear-test.js diff --git a/README.md b/README.md index 53f71865..8f9d9d83 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,19 @@ In addition to range options, `iterator()` takes the following options: Lastly, an implementation is free to add its own options. +### `db.clear([options, ]callback)` + +**This method is experimental. Not all implementations support it yet.** + +Delete all entries or a range. Not guaranteed to be atomic. Accepts the following range options (with the same rules as on iterators): + +- `gt` (greater than), `gte` (greater than or equal) define the lower bound of the range to be deleted. Only entries where the key is greater than (or equal to) this option will be included in the range. When `reverse=true` the order will be reversed, but the entries deleted will be the same. +- `lt` (less than), `lte` (less than or equal) define the higher bound of the range to be deleted. Only entries where the key is less than (or equal to) this option will be included in the range. When `reverse=true` the order will be reversed, but the entries deleted will be the same. +- `reverse` _(boolean, default: `false`)_: delete entries in reverse order. Only effective in combination with `limit`, to remove the last N records. +- `limit` _(number, default: `-1`)_: limit the number of entries to be deleted. This number represents a _maximum_ number of entries and may not be reached if you get to the end of the range first. A value of `-1` means there is no limit. When `reverse=true` the entries with the highest keys will be deleted instead of the lowest keys. + +If no options are provided, all entries will be deleted. The `callback` function will be called with no arguments if the operation was successful or with an `Error` if it failed for any reason. + ### `chainedBatch` #### `chainedBatch.put(key, value)` @@ -356,6 +369,18 @@ The default `_iterator()` returns a noop `AbstractIterator` instance. The protot The `options` object will always have the following properties: `reverse`, `keys`, `values`, `limit`, `keyAsBuffer` and `valueAsBuffer`. +### `db._clear(options, callback)` + +**This method is experimental and optional for the time being. To enable its tests, set the [`clear` option of the test suite](#excluding-tests) to `true`.** + +Delete all entries or a range. Does not have to be atomic. It is recommended (and possibly mandatory in the future) to operate on a snapshot so that writes scheduled after a call to `clear()` will not be affected. + +The default `_clear()` uses `_iterator()` and `_del()` to provide a reasonable fallback, but requires binary key support. It is _recommended_ to implement `_clear()` with more performant primitives than `_iterator()` and `_del()` if the underlying storage has such primitives. Implementations that don't support binary keys _must_ implement their own `_clear()`. + +Implementations that wrap another `db` can typically forward the `_clear()` call to that `db`, having transformed range options if necessary. + +The `options` object will always have the following properties: `reverse` and `limit`. + ### `iterator = AbstractIterator(db)` The first argument to this constructor must be an instance of your `AbstractLevelDOWN` implementation. The constructor will set `iterator.db` which is used to access `db._serialize*` and ensures that `db` will not be garbage collected in case there are no other references to it. @@ -442,6 +467,7 @@ This also serves as a signal to users of your implementation. The following opti - `bufferKeys`: set to `false` if binary keys are not supported by the underlying storage - `seek`: set to `false` if your `iterator` does not implement `_seek` +- `clear`: defaults to `false` until a next major release. Set to `true` if your implementation either implements `_clear()` itself or is suitable to use the default implementation of `_clear()` (which requires binary key support). - `snapshots`: set to `false` if any of the following is true: - Reads don't operate on a [snapshot](#iterator) - Snapshots are created asynchronously diff --git a/abstract-leveldown.js b/abstract-leveldown.js index fc4ab660..cd744c24 100644 --- a/abstract-leveldown.js +++ b/abstract-leveldown.js @@ -183,6 +183,52 @@ AbstractLevelDOWN.prototype._batch = function (array, options, callback) { process.nextTick(callback) } +AbstractLevelDOWN.prototype.clear = function (options, callback) { + if (typeof options === 'function') { + callback = options + } else if (typeof callback !== 'function') { + throw new Error('clear() requires a callback argument') + } + + options = cleanRangeOptions(this, options) + options.reverse = !!options.reverse + options.limit = 'limit' in options ? options.limit : -1 + + this._clear(options, callback) +} + +AbstractLevelDOWN.prototype._clear = function (options, callback) { + // Avoid setupIteratorOptions, would serialize range options a second time. + options.keys = true + options.values = false + options.keyAsBuffer = true + options.valueAsBuffer = true + + var iterator = this._iterator(options) + var emptyOptions = {} + var self = this + + var next = function (err) { + if (err) { + return iterator.end(function () { + callback(err) + }) + } + + iterator.next(function (err, key) { + if (err) return next(err) + if (key === undefined) return iterator.end(callback) + + // This could be optimized by using a batch, but the default _clear + // is not meant to be fast. Implementations have more room to optimize + // if they override _clear. Note: using _del bypasses key serialization. + self._del(key, emptyOptions, next) + }) + } + + next() +} + AbstractLevelDOWN.prototype._setupIteratorOptions = function (options) { options = cleanRangeOptions(this, options) diff --git a/test/clear-range-test.js b/test/clear-range-test.js new file mode 100644 index 00000000..21586318 --- /dev/null +++ b/test/clear-range-test.js @@ -0,0 +1,258 @@ +var concat = require('level-concat-iterator') + +var data = (function () { + var d = [] + var i = 0 + var k + for (; i < 100; i++) { + k = (i < 10 ? '0' : '') + i + d.push({ + key: k, + value: String(Math.random()) + }) + } + return d +}()) + +exports.setUp = function (test, testCommon) { + test('setUp common', testCommon.setUp) +} + +exports.range = function (test, testCommon) { + function rangeTest (name, opts, expected) { + test('db#clear() with ' + name, function (t) { + prepare(t, function (db) { + db.clear(opts, function (err) { + t.ifError(err, 'no clear error') + verify(t, db, expected) + }) + }) + }) + } + + function prepare (t, callback) { + var db = testCommon.factory() + + db.open(function (err) { + t.ifError(err, 'no open error') + + db.batch(data.map(function (d) { + return { + type: 'put', + key: d.key, + value: d.value + } + }), function (err) { + t.ifError(err, 'no batch error') + callback(db) + }) + }) + } + + function verify (t, db, expected) { + var it = db.iterator({ keyAsBuffer: false, valueAsBuffer: false }) + + concat(it, function (err, result) { + t.ifError(err, 'no concat error') + t.is(result.length, expected.length, 'correct number of entries') + t.same(result, expected) + + db.close(t.end.bind(t)) + }) + } + + function exclude (data, start, end, expectedLength) { + data = data.slice() + var removed = data.splice(start, end - start + 1) // Inclusive + if (expectedLength != null) checkLength(removed, expectedLength) + return data + } + + // For sanity checks on test arguments + function checkLength (arr, length) { + if (arr.length !== length) { + throw new RangeError('Expected ' + length + ' elements, got ' + arr.length) + } + + return arr + } + + rangeTest('full range', {}, []) + + // Reversing has no effect without limit + rangeTest('reverse=true', { + reverse: true + }, []) + + rangeTest('gte=00', { + gte: '00' + }, []) + + rangeTest('gte=50', { + gte: '50' + }, data.slice(0, 50)) + + rangeTest('lte=50 and reverse=true', { + lte: '50', + reverse: true + }, data.slice(51)) + + rangeTest('gte=49.5 (midway)', { + gte: '49.5' + }, data.slice(0, 50)) + + rangeTest('gte=49999 (midway)', { + gte: '49999' + }, data.slice(0, 50)) + + rangeTest('lte=49.5 (midway) and reverse=true', { + lte: '49.5', + reverse: true + }, data.slice(50)) + + rangeTest('lt=49.5 (midway) and reverse=true', { + lt: '49.5', + reverse: true + }, data.slice(50)) + + rangeTest('lt=50 and reverse=true', { + lt: '50', + reverse: true + }, data.slice(50)) + + rangeTest('lte=50', { + lte: '50' + }, data.slice(51)) + + rangeTest('lte=50.5 (midway)', { + lte: '50.5' + }, data.slice(51)) + + rangeTest('lte=50555 (midway)', { + lte: '50555' + }, data.slice(51)) + + rangeTest('lt=50555 (midway)', { + lt: '50555' + }, data.slice(51)) + + rangeTest('gte=50.5 (midway) and reverse=true', { + gte: '50.5', + reverse: true + }, data.slice(0, 51)) + + rangeTest('gt=50.5 (midway) and reverse=true', { + gt: '50.5', + reverse: true + }, data.slice(0, 51)) + + rangeTest('gt=50 and reverse=true', { + gt: '50', + reverse: true + }, data.slice(0, 51)) + + // Starting key is actually '00' so it should avoid it + rangeTest('lte=0', { + lte: '0' + }, data) + + // Starting key is actually '00' so it should avoid it + rangeTest('lt=0', { + lt: '0' + }, data) + + rangeTest('gte=30 and lte=70', { + gte: '30', + lte: '70' + }, exclude(data, 30, 70)) + + rangeTest('gt=29 and lt=71', { + gt: '29', + lt: '71' + }, exclude(data, 30, 70)) + + rangeTest('gte=30 and lte=70 and reverse=true', { + lte: '70', + gte: '30', + reverse: true + }, exclude(data, 30, 70)) + + rangeTest('gt=29 and lt=71 and reverse=true', { + lt: '71', + gt: '29', + reverse: true + }, exclude(data, 30, 70)) + + rangeTest('limit=20', { + limit: 20 + }, data.slice(20)) + + rangeTest('limit=20 and gte=20', { + limit: 20, + gte: '20' + }, exclude(data, 20, 39, 20)) + + rangeTest('limit=20 and reverse=true', { + limit: 20, + reverse: true + }, data.slice(0, -20)) + + rangeTest('limit=20 and lte=79 and reverse=true', { + limit: 20, + lte: '79', + reverse: true + }, exclude(data, 60, 79, 20)) + + rangeTest('limit=-1 should clear whole database', { + limit: -1 + }, []) + + rangeTest('limit=0 should not clear anything', { + limit: 0 + }, data) + + rangeTest('lte after limit', { + limit: 20, + lte: '50' + }, data.slice(20)) + + rangeTest('lte before limit', { + limit: 50, + lte: '19' + }, data.slice(20)) + + rangeTest('gte after database end', { + gte: '9a' + }, data) + + rangeTest('gt after database end', { + gt: '9a' + }, data) + + rangeTest('lte after database end and reverse=true', { + lte: '9a', + reverse: true + }, []) + + rangeTest('lte and gte after database and reverse=true', { + lte: '9b', + gte: '9a', + reverse: true + }, data) + + rangeTest('lt and gt after database and reverse=true', { + lt: '9b', + gt: '9a', + reverse: true + }, data) +} + +exports.tearDown = function (test, testCommon) { + test('tearDown', testCommon.tearDown) +} + +exports.all = function (test, testCommon) { + exports.setUp(test, testCommon) + exports.range(test, testCommon) + exports.tearDown(test, testCommon) +} diff --git a/test/clear-test.js b/test/clear-test.js new file mode 100644 index 00000000..d338cad0 --- /dev/null +++ b/test/clear-test.js @@ -0,0 +1,83 @@ +var concat = require('level-concat-iterator') +var db + +exports.setUp = function (test, testCommon) { + test('setUp common', testCommon.setUp) + test('setUp db', function (t) { + db = testCommon.factory() + db.open(t.end.bind(t)) + }) +} + +exports.args = function (test, testCommon) { + test('test argument-less clear() throws', function (t) { + t.throws( + db.clear.bind(db), + /Error: clear\(\) requires a callback argument/, + 'no-arg clear() throws' + ) + t.end() + }) +} + +exports.clear = function (test, testCommon) { + makeTest('string', ['a', 'b']) + + if (testCommon.bufferKeys) { + makeTest('buffer', [Buffer.from('a'), Buffer.from('b')]) + makeTest('mixed', [Buffer.from('a'), 'b']) + + // These keys would be equal when compared as utf8 strings + makeTest('non-utf8 buffer', [Buffer.from('80', 'hex'), Buffer.from('c0', 'hex')]) + } + + function makeTest (type, keys) { + test('test simple clear() on ' + type + ' keys', function (t) { + t.plan(8) + + var db = testCommon.factory() + var ops = keys.map(function (key) { + return { type: 'put', key: key, value: 'foo' } + }) + + db.open(function (err) { + t.ifError(err, 'no open error') + + db.batch(ops, function (err) { + t.ifError(err, 'no batch error') + + concat(db.iterator(), function (err, entries) { + t.ifError(err, 'no concat error') + t.is(entries.length, keys.length, 'has entries') + + db.clear(function (err) { + t.ifError(err, 'no clear error') + + concat(db.iterator(), function (err, entries) { + t.ifError(err, 'no concat error') + t.is(entries.length, 0, 'has no entries') + + db.close(function (err) { + t.ifError(err, 'no close error') + }) + }) + }) + }) + }) + }) + }) + } +} + +exports.tearDown = function (test, testCommon) { + test('tearDown', function (t) { + db.close(testCommon.tearDown.bind(null, t)) + }) +} + +exports.all = function (test, testCommon) { + exports.setUp(test, testCommon) + exports.args(test, testCommon) + exports.clear(test, testCommon) + exports.tearDown(test, testCommon) +} diff --git a/test/common.js b/test/common.js index 94241fca..1bd13563 100644 --- a/test/common.js +++ b/test/common.js @@ -1,6 +1,9 @@ +var warned = false + function testCommon (options) { var factory = options.factory var test = options.test + var clear = !!options.clear if (typeof factory !== 'function') { throw new TypeError('factory must be a function') @@ -10,6 +13,15 @@ function testCommon (options) { throw new TypeError('test must be a function') } + if (!clear && !warned) { + warned = true + warn( + 'A next major release of abstract-leveldown will make support of ' + + 'clear() mandatory. Prepare by enabling the tests and implementing a ' + + 'custom _clear() if necessary. See the README for details.' + ) + } + return { test: test, factory: factory, @@ -19,7 +31,16 @@ function testCommon (options) { createIfMissing: options.createIfMissing !== false, errorIfExists: options.errorIfExists !== false, snapshots: options.snapshots !== false, - seek: options.seek !== false + seek: options.seek !== false, + clear: clear + } +} + +function warn (msg) { + if (typeof process !== 'undefined' && process && process.emitWarning) { + process.emitWarning(msg) + } else if (typeof console !== 'undefined' && console && console.warn) { + console.warn('Warning: ' + msg) } } diff --git a/test/index.js b/test/index.js index 9e399559..9c2fa35a 100644 --- a/test/index.js +++ b/test/index.js @@ -38,6 +38,11 @@ function suite (options) { } else { require('./iterator-no-snapshot-test').all(test, testCommon) } + + if (testCommon.clear) { + require('./clear-test').all(test, testCommon) + require('./clear-range-test').all(test, testCommon) + } } suite.common = common diff --git a/test/self.js b/test/self.js index a2a83d97..18e156c0 100644 --- a/test/self.js +++ b/test/self.js @@ -9,11 +9,15 @@ var AbstractChainedBatch = require('../').AbstractChainedBatch var testCommon = require('./common')({ test: test, + clear: true, factory: function () { return new AbstractLevelDOWN() } }) +var rangeOptions = ['gt', 'gte', 'lt', 'lte'] +var legacyRangeOptions = ['start', 'end'] + // Test the suite itself as well as the default implementation, // excluding noop operations that can't pass the test suite. @@ -67,6 +71,13 @@ require('./iterator-seek-test').setUp(test, testCommon) require('./iterator-seek-test').sequence(test, testCommon) require('./iterator-seek-test').tearDown(test, testCommon) +require('./clear-test').setUp(test, testCommon) +require('./clear-test').args(test, testCommon) +require('./clear-test').tearDown(test, testCommon) + +require('./clear-range-test').setUp(test, testCommon) +require('./clear-range-test').tearDown(test, testCommon) + function implement (ctor, methods) { function Test () { ctor.apply(this, arguments) @@ -374,7 +385,7 @@ test('test AbstractChainedBatch expects a db', function (t) { } }) -test('test write() extensibility', function (t) { +test('test AbstractChainedBatch#write() extensibility', function (t) { var spy = sinon.spy() var spycb = sinon.spy() var Test = implement(AbstractChainedBatch, { _write: spy }) @@ -394,7 +405,7 @@ test('test write() extensibility', function (t) { t.end() }) -test('test write() extensibility with null options', function (t) { +test('test AbstractChainedBatch#write() extensibility with null options', function (t) { var spy = sinon.spy() var Test = implement(AbstractChainedBatch, { _write: spy }) var test = new Test({ test: true }) @@ -406,7 +417,7 @@ test('test write() extensibility with null options', function (t) { t.end() }) -test('test write() extensibility with options', function (t) { +test('test AbstractChainedBatch#write() extensibility with options', function (t) { var spy = sinon.spy() var Test = implement(AbstractChainedBatch, { _write: spy }) var test = new Test({ test: true }) @@ -418,7 +429,7 @@ test('test write() extensibility with options', function (t) { t.end() }) -test('test put() extensibility', function (t) { +test('test AbstractChainedBatch#put() extensibility', function (t) { var spy = sinon.spy() var expectedKey = 'key' var expectedValue = 'value' @@ -435,7 +446,7 @@ test('test put() extensibility', function (t) { t.end() }) -test('test del() extensibility', function (t) { +test('test AbstractChainedBatch#del() extensibility', function (t) { var spy = sinon.spy() var expectedKey = 'key' var Test = implement(AbstractChainedBatch, { _del: spy }) @@ -450,7 +461,7 @@ test('test del() extensibility', function (t) { t.end() }) -test('test clear() extensibility', function (t) { +test('test AbstractChainedBatch#clear() extensibility', function (t) { var spy = sinon.spy() var Test = implement(AbstractChainedBatch, { _clear: spy }) var test = new Test(testCommon.factory()) @@ -479,8 +490,8 @@ test('test iterator() extensibility', function (t) { test.iterator({ options: 1 }) - t.equal(spy.callCount, 1, 'got _close() call') - t.equal(spy.getCall(0).thisValue, test, '`this` on _close() was correct') + t.equal(spy.callCount, 1, 'got _iterator() call') + t.equal(spy.getCall(0).thisValue, test, '`this` on _iterator() was correct') t.equal(spy.getCall(0).args.length, 1, 'got one arguments') t.deepEqual(spy.getCall(0).args[0], expectedOptions, 'got expected options argument') t.end() @@ -494,7 +505,7 @@ test('test AbstractIterator extensibility', function (t) { t.end() }) -test('test next() extensibility', function (t) { +test('test AbstractIterator#next() extensibility', function (t) { var spy = sinon.spy() var spycb = sinon.spy() var Test = implement(AbstractIterator, { _next: spy }) @@ -513,7 +524,7 @@ test('test next() extensibility', function (t) { t.end() }) -test('test end() extensibility', function (t) { +test('test AbstractIterator#end() extensibility', function (t) { var spy = sinon.spy() var expectedCb = function () {} var Test = implement(AbstractIterator, { _end: spy }) @@ -528,6 +539,35 @@ test('test end() extensibility', function (t) { t.end() }) +test('test clear() extensibility', function (t) { + var spy = sinon.spy() + var Test = implement(AbstractLevelDOWN, { _clear: spy }) + var db = new Test() + var callback = function () {} + + call([callback], { reverse: false, limit: -1 }) + call([null, callback], { reverse: false, limit: -1 }) + call([undefined, callback], { reverse: false, limit: -1 }) + call([{ custom: 1 }, callback], { custom: 1, reverse: false, limit: -1 }) + call([{ reverse: true, limit: 0 }, callback], { reverse: true, limit: 0 }) + call([{ reverse: 1 }, callback], { reverse: true, limit: -1 }) + call([{ reverse: null }, callback], { reverse: false, limit: -1 }) + + function call (args, expectedOptions) { + db.clear.apply(db, args) + + t.is(spy.callCount, 1, 'got _clear() call') + t.is(spy.getCall(0).thisValue, db, '`this` on _clear() was correct') + t.is(spy.getCall(0).args.length, 2, 'got two arguments') + t.same(spy.getCall(0).args[0], expectedOptions, 'got expected options argument') + t.is(spy.getCall(0).args[1], callback, 'got expected callback argument') + + spy.resetHistory() + } + + t.end() +}) + test('test serialization extensibility (put)', function (t) { t.plan(5) @@ -752,6 +792,53 @@ test('test serialization extensibility (iterator seek)', function (t) { t.equal(spy.getCall(0).args[0], 'serialized', 'got expected target argument') }) +test('test serialization extensibility (clear range options)', function (t) { + t.plan(rangeOptions.length * 2) + + rangeOptions.forEach(function (key) { + var Test = implement(AbstractLevelDOWN, { + _serializeKey: function (key) { + t.is(key, 'input') + return 'output' + }, + _clear: function (options, callback) { + t.is(options[key], 'output') + } + }) + + var db = new Test() + var options = {} + + options[key] = 'input' + db.clear(options, function () {}) + }) +}) + +test('clear() does not delete empty or nullish range options', function (t) { + var rangeValues = [Buffer.alloc(0), '', null, undefined] + + t.plan(rangeOptions.length * rangeValues.length) + + rangeValues.forEach(function (value) { + var Test = implement(AbstractLevelDOWN, { + _clear: function (options, callback) { + rangeOptions.forEach(function (key) { + t.ok(key in options, key + ' option should not be deleted') + }) + } + }) + + var db = new Test() + var options = {} + + rangeOptions.forEach(function (key) { + options[key] = value + }) + + db.clear(options, function () {}) + }) +}) + test('.status', function (t) { t.plan(5) @@ -843,7 +930,7 @@ test('.status', function (t) { }) test('_setupIteratorOptions', function (t) { - var keys = 'start end gt gte lt lte'.split(' ') + var keys = legacyRangeOptions.concat(rangeOptions) var db = new AbstractLevelDOWN() function setupOptions (constrFn) { @@ -856,7 +943,7 @@ test('_setupIteratorOptions', function (t) { function verifyOptions (t, options) { keys.forEach(function (key) { - t.ok(key in options, 'property should not be deleted') + t.ok(key in options, key + ' option should not be deleted') }) t.end() }