Add clear() method to delete all entries or a range (#310)

README.md

@@ -205,6 +205,19 @@ In addition to range options, `iterator()` takes the following options:
 
 Lastly, an implementation is free to add its own options.
 
+### `db.clear([options, ]callback)`
+
+**This method is experimental. Not all implementations support it yet.**
+
+Delete all entries or a range. Not guaranteed to be atomic. Accepts the following range options (with the same rules as on iterators):
+
+- `gt` (greater than), `gte` (greater than or equal) define the lower bound of the range to be deleted. Only entries where the key is greater than (or equal to) this option will be included in the range. When `reverse=true` the order will be reversed, but the entries deleted will be the same.
+- `lt` (less than), `lte` (less than or equal) define the higher bound of the range to be deleted. Only entries where the key is less than (or equal to) this option will be included in the range. When `reverse=true` the order will be reversed, but the entries deleted will be the same.
+- `reverse` _(boolean, default: `false`)_: delete entries in reverse order. Only effective in combination with `limit`, to remove the last N records.
+- `limit` _(number, default: `-1`)_: limit the number of entries to be deleted. This number represents a _maximum_ number of entries and may not be reached if you get to the end of the range first. A value of `-1` means there is no limit. When `reverse=true` the entries with the highest keys will be deleted instead of the lowest keys.
+
+If no options are provided, all entries will be deleted. The `callback` function will be called with no arguments if the operation was successful or with an `Error` if it failed for any reason.
+
 ### `chainedBatch`
 
 #### `chainedBatch.put(key, value)`
@@ -356,6 +369,18 @@ The default `_iterator()` returns a noop `AbstractIterator` instance. The protot
 
 The `options` object will always have the following properties: `reverse`, `keys`, `values`, `limit`, `keyAsBuffer` and `valueAsBuffer`.
 
+### `db._clear(options, callback)`
+
+**This method is experimental and optional for the time being. To enable its tests, set the [`clear` option of the test suite](#excluding-tests) to `true`.**
+
+Delete all entries or a range. Does not have to be atomic. It is recommended (and possibly mandatory in the future) to operate on a snapshot so that writes scheduled after a call to `clear()` will not be affected.
+
+The default `_clear()` uses `_iterator()` and `_del()` to provide a reasonable fallback, but requires binary key support. It is _recommended_ to implement `_clear()` with more performant primitives than `_iterator()` and `_del()` if the underlying storage has such primitives. Implementations that don't support binary keys _must_ implement their own `_clear()`.
+
+Implementations that wrap another `db` can typically forward the `_clear()` call to that `db`, having transformed range options if necessary.
+
+The `options` object will always have the following properties: `reverse` and `limit`.
+
 ### `iterator = AbstractIterator(db)`
 
 The first argument to this constructor must be an instance of your `AbstractLevelDOWN` implementation. The constructor will set `iterator.db` which is used to access `db._serialize*` and ensures that `db` will not be garbage collected in case there are no other references to it.
@@ -442,6 +467,7 @@ This also serves as a signal to users of your implementation. The following opti
 
 - `bufferKeys`: set to `false` if binary keys are not supported by the underlying storage
 - `seek`: set to `false` if your `iterator` does not implement `_seek`
+- `clear`: defaults to `false` until a next major release. Set to `true` if your implementation either implements `_clear()` itself or is suitable to use the default implementation of `_clear()` (which requires binary key support).
 - `snapshots`: set to `false` if any of the following is true:
   - Reads don't operate on a [snapshot](#iterator)
   - Snapshots are created asynchronously

abstract-leveldown.js

@@ -183,6 +183,52 @@ AbstractLevelDOWN.prototype._batch = function (array, options, callback) {
   process.nextTick(callback)
 }
 
+AbstractLevelDOWN.prototype.clear = function (options, callback) {
+  if (typeof options === 'function') {
+    callback = options
+  } else if (typeof callback !== 'function') {
+    throw new Error('clear() requires a callback argument')
+  }
+
+  options = cleanRangeOptions(this, options)
+  options.reverse = !!options.reverse
+  options.limit = 'limit' in options ? options.limit : -1
+
+  this._clear(options, callback)
+}
+
+AbstractLevelDOWN.prototype._clear = function (options, callback) {
+  // Avoid setupIteratorOptions, would serialize range options a second time.
+  options.keys = true
+  options.values = false
+  options.keyAsBuffer = true
+  options.valueAsBuffer = true
+
+  var iterator = this._iterator(options)
+  var emptyOptions = {}
+  var self = this
+
+  var next = function (err) {
+    if (err) {
+      return iterator.end(function () {
+        callback(err)
+      })
+    }
+
+    iterator.next(function (err, key) {
+      if (err) return next(err)
+      if (key === undefined) return iterator.end(callback)
+
+      // This could be optimized by using a batch, but the default _clear
+      // is not meant to be fast. Implementations have more room to optimize
+      // if they override _clear. Note: using _del bypasses key serialization.
+      self._del(key, emptyOptions, next)
+    })
+  }
+
+  next()
+}
+
 AbstractLevelDOWN.prototype._setupIteratorOptions = function (options) {
   options = cleanRangeOptions(this, options)
 

test/clear-range-test.js

@@ -0,0 +1,258 @@
+var concat = require('level-concat-iterator')
+
+var data = (function () {
+  var d = []
+  var i = 0
+  var k
+  for (; i < 100; i++) {
+    k = (i < 10 ? '0' : '') + i
+    d.push({
+      key: k,
+      value: String(Math.random())
+    })
+  }
+  return d
+}())
+
+exports.setUp = function (test, testCommon) {
+  test('setUp common', testCommon.setUp)
+}
+
+exports.range = function (test, testCommon) {
+  function rangeTest (name, opts, expected) {
+    test('db#clear() with ' + name, function (t) {
+      prepare(t, function (db) {
+        db.clear(opts, function (err) {
+          t.ifError(err, 'no clear error')
+          verify(t, db, expected)
+        })
+      })
+    })
+  }
+
+  function prepare (t, callback) {
+    var db = testCommon.factory()
+
+    db.open(function (err) {
+      t.ifError(err, 'no open error')
+
+      db.batch(data.map(function (d) {
+        return {
+          type: 'put',
+          key: d.key,
+          value: d.value
+        }
+      }), function (err) {
+        t.ifError(err, 'no batch error')
+        callback(db)
+      })
+    })
+  }
+
+  function verify (t, db, expected) {
+    var it = db.iterator({ keyAsBuffer: false, valueAsBuffer: false })
+
+    concat(it, function (err, result) {
+      t.ifError(err, 'no concat error')
+      t.is(result.length, expected.length, 'correct number of entries')
+      t.same(result, expected)
+
+      db.close(t.end.bind(t))
+    })
+  }
+
+  function exclude (data, start, end, expectedLength) {
+    data = data.slice()
+    var removed = data.splice(start, end - start + 1) // Inclusive
+    if (expectedLength != null) checkLength(removed, expectedLength)
+    return data
+  }
+
+  // For sanity checks on test arguments
+  function checkLength (arr, length) {
+    if (arr.length !== length) {
+      throw new RangeError('Expected ' + length + ' elements, got ' + arr.length)
+    }
+
+    return arr
+  }
+
+  rangeTest('full range', {}, [])
+
+  // Reversing has no effect without limit
+  rangeTest('reverse=true', {
+    reverse: true
+  }, [])
+
+  rangeTest('gte=00', {
+    gte: '00'
+  }, [])
+
+  rangeTest('gte=50', {
+    gte: '50'
+  }, data.slice(0, 50))
+
+  rangeTest('lte=50 and reverse=true', {
+    lte: '50',
+    reverse: true
+  }, data.slice(51))
+
+  rangeTest('gte=49.5 (midway)', {
+    gte: '49.5'
+  }, data.slice(0, 50))
+
+  rangeTest('gte=49999 (midway)', {
+    gte: '49999'
+  }, data.slice(0, 50))
+
+  rangeTest('lte=49.5 (midway) and reverse=true', {
+    lte: '49.5',
+    reverse: true
+  }, data.slice(50))
+
+  rangeTest('lt=49.5 (midway) and reverse=true', {
+    lt: '49.5',
+    reverse: true
+  }, data.slice(50))
+
+  rangeTest('lt=50 and reverse=true', {
+    lt: '50',
+    reverse: true
+  }, data.slice(50))
+
+  rangeTest('lte=50', {
+    lte: '50'
+  }, data.slice(51))
+
+  rangeTest('lte=50.5 (midway)', {
+    lte: '50.5'
+  }, data.slice(51))
+
+  rangeTest('lte=50555 (midway)', {
+    lte: '50555'
+  }, data.slice(51))
+
+  rangeTest('lt=50555 (midway)', {
+    lt: '50555'
+  }, data.slice(51))
+
+  rangeTest('gte=50.5 (midway) and reverse=true', {
+    gte: '50.5',
+    reverse: true
+  }, data.slice(0, 51))
+
+  rangeTest('gt=50.5 (midway) and reverse=true', {
+    gt: '50.5',
+    reverse: true
+  }, data.slice(0, 51))
+
+  rangeTest('gt=50 and reverse=true', {
+    gt: '50',
+    reverse: true
+  }, data.slice(0, 51))
+
+  // Starting key is actually '00' so it should avoid it
+  rangeTest('lte=0', {
+    lte: '0'
+  }, data)
+
+  // Starting key is actually '00' so it should avoid it
+  rangeTest('lt=0', {
+    lt: '0'
+  }, data)
+
+  rangeTest('gte=30 and lte=70', {
+    gte: '30',
+    lte: '70'
+  }, exclude(data, 30, 70))
+
+  rangeTest('gt=29 and lt=71', {
+    gt: '29',
+    lt: '71'
+  }, exclude(data, 30, 70))
+
+  rangeTest('gte=30 and lte=70 and reverse=true', {
+    lte: '70',
+    gte: '30',
+    reverse: true
+  }, exclude(data, 30, 70))
+
+  rangeTest('gt=29 and lt=71 and reverse=true', {
+    lt: '71',
+    gt: '29',
+    reverse: true
+  }, exclude(data, 30, 70))
+
+  rangeTest('limit=20', {
+    limit: 20
+  }, data.slice(20))
+
+  rangeTest('limit=20 and gte=20', {
+    limit: 20,
+    gte: '20'
+  }, exclude(data, 20, 39, 20))
+
+  rangeTest('limit=20 and reverse=true', {
+    limit: 20,
+    reverse: true
+  }, data.slice(0, -20))
+
+  rangeTest('limit=20 and lte=79 and reverse=true', {
+    limit: 20,
+    lte: '79',
+    reverse: true
+  }, exclude(data, 60, 79, 20))
+
+  rangeTest('limit=-1 should clear whole database', {
+    limit: -1
+  }, [])
+
+  rangeTest('limit=0 should not clear anything', {
+    limit: 0
+  }, data)
+
+  rangeTest('lte after limit', {
+    limit: 20,
+    lte: '50'
+  }, data.slice(20))
+
+  rangeTest('lte before limit', {
+    limit: 50,
+    lte: '19'
+  }, data.slice(20))
+
+  rangeTest('gte after database end', {
+    gte: '9a'
+  }, data)
+
+  rangeTest('gt after database end', {
+    gt: '9a'
+  }, data)
+
+  rangeTest('lte after database end and reverse=true', {
+    lte: '9a',
+    reverse: true
+  }, [])
+
+  rangeTest('lte and gte after database and reverse=true', {
+    lte: '9b',
+    gte: '9a',
+    reverse: true
+  }, data)
+
+  rangeTest('lt and gt after database and reverse=true', {
+    lt: '9b',
+    gt: '9a',
+    reverse: true
+  }, data)
+}
+
+exports.tearDown = function (test, testCommon) {
+  test('tearDown', testCommon.tearDown)
+}
+
+exports.all = function (test, testCommon) {
+  exports.setUp(test, testCommon)
+  exports.range(test, testCommon)
+  exports.tearDown(test, testCommon)
+}