From 47e0137fc5244d291d696296ef23da32ff7d8a39 Mon Sep 17 00:00:00 2001 From: Vincent Weevers Date: Fri, 1 Oct 2021 16:42:19 +0200 Subject: [PATCH] Optimize `db.clear()` Ref https://github.com/Level/community/issues/79 --- leveldown.js | 17 +- messages.js | 609 ++++++++++++++++++++++++++++++++++----------------- schema.proto | 14 ++ server.js | 11 +- 4 files changed, 453 insertions(+), 198 deletions(-) diff --git a/leveldown.js b/leveldown.js index 3cdc0f5..e59482e 100644 --- a/leveldown.js +++ b/leveldown.js @@ -14,7 +14,8 @@ const ENCODERS = [ messages.Put, messages.Delete, messages.Batch, - messages.Iterator + messages.Iterator, + messages.Clear ] const DECODERS = [ @@ -217,6 +218,20 @@ Multilevel.prototype._batch = function (batch, opts, cb) { this._write(req) } +Multilevel.prototype._clear = function (opts, cb) { + if (this._db) return this._db._clear(opts, cb) + + const req = { + tag: 5, + id: 0, + options: opts, + callback: cb || noop + } + + req.id = this._requests.add(req) + this._write(req) +} + Multilevel.prototype._write = function (req) { if (this._requests.length + this._iterators.length === 1) ref(this._ref) const enc = ENCODERS[req.tag] diff --git a/messages.js b/messages.js index fde58f6..d19ca16 100644 --- a/messages.js +++ b/messages.js @@ -1,77 +1,85 @@ -// This file is auto generated by the protocol-buffers compiler - -/* eslint-disable quotes */ -/* eslint-disable indent */ -/* eslint-disable no-redeclare */ -/* eslint-disable camelcase */ - -// Remember to `npm install --save protocol-buffers-encodings` -var encodings = require('protocol-buffers-encodings') -var varint = encodings.varint -var skip = encodings.skip - -var Get = exports.Get = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Put = exports.Put = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Delete = exports.Delete = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Batch = exports.Batch = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Iterator = exports.Iterator = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Callback = exports.Callback = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var IteratorData = exports.IteratorData = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -defineGet() -definePut() -defineDelete() -defineBatch() -defineIterator() -defineCallback() -defineIteratorData() - -function defineGet () { - Get.encodingLength = encodingLength - Get.encode = encode - Get.decode = decode - +// This file is auto generated by the protocol-buffers compiler + +/* eslint-disable quotes */ +/* eslint-disable indent */ +/* eslint-disable no-redeclare */ +/* eslint-disable camelcase */ + +// Remember to `npm install --save protocol-buffers-encodings` +var encodings = require('protocol-buffers-encodings') +var varint = encodings.varint +var skip = encodings.skip + +var Get = exports.Get = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Put = exports.Put = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Delete = exports.Delete = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Batch = exports.Batch = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Clear = exports.Clear = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Iterator = exports.Iterator = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Callback = exports.Callback = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var IteratorData = exports.IteratorData = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +defineGet() +definePut() +defineDelete() +defineBatch() +defineClear() +defineIterator() +defineCallback() +defineIteratorData() + +function defineGet () { + Get.encodingLength = encodingLength + Get.encode = encode + Get.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -81,8 +89,8 @@ function defineGet () { var len = encodings.bytes.encodingLength(obj.key) length += 1 + len return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -97,8 +105,8 @@ function defineGet () { offset += encodings.bytes.encode.bytes encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -134,14 +142,14 @@ function defineGet () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function definePut () { - Put.encodingLength = encodingLength - Put.encode = encode - Put.decode = decode - + } +} + +function definePut () { + Put.encodingLength = encodingLength + Put.encode = encode + Put.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -155,8 +163,8 @@ function definePut () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -176,8 +184,8 @@ function definePut () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -218,14 +226,14 @@ function definePut () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineDelete () { - Delete.encodingLength = encodingLength - Delete.encode = encode - Delete.decode = decode - + } +} + +function defineDelete () { + Delete.encodingLength = encodingLength + Delete.encode = encode + Delete.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -235,8 +243,8 @@ function defineDelete () { var len = encodings.bytes.encodingLength(obj.key) length += 1 + len return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -251,8 +259,8 @@ function defineDelete () { offset += encodings.bytes.encode.bytes encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -288,24 +296,24 @@ function defineDelete () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineBatch () { - var Operation = Batch.Operation = { - buffer: true, - encodingLength: null, - encode: null, - decode: null - } - - defineOperation() - - function defineOperation () { - Operation.encodingLength = encodingLength - Operation.encode = encode - Operation.decode = decode - + } +} + +function defineBatch () { + var Operation = Batch.Operation = { + buffer: true, + encodingLength: null, + encode: null, + decode: null + } + + defineOperation() + + function defineOperation () { + Operation.encodingLength = encodingLength + Operation.encode = encode + Operation.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.type)) throw new Error("type is required") @@ -319,8 +327,8 @@ function defineBatch () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -340,8 +348,8 @@ function defineBatch () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -382,13 +390,13 @@ function defineBatch () { offset = skip(prefix & 7, buf, offset) } } - } - } - - Batch.encodingLength = encodingLength - Batch.encode = encode - Batch.decode = decode - + } + } + + Batch.encodingLength = encodingLength + Batch.encode = encode + Batch.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -403,8 +411,8 @@ function defineBatch () { } } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -425,8 +433,8 @@ function defineBatch () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -462,24 +470,233 @@ function defineBatch () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineIterator () { - var Options = Iterator.Options = { - buffer: true, - encodingLength: null, - encode: null, - decode: null - } - - defineOptions() - - function defineOptions () { - Options.encodingLength = encodingLength - Options.encode = encode - Options.decode = decode - + } +} + +function defineClear () { + var ClearOptions = Clear.ClearOptions = { + buffer: true, + encodingLength: null, + encode: null, + decode: null + } + + defineClearOptions() + + function defineClearOptions () { + ClearOptions.encodingLength = encodingLength + ClearOptions.encode = encode + ClearOptions.decode = decode + + function encodingLength (obj) { + var length = 0 + if (defined(obj.gt)) { + var len = encodings.bytes.encodingLength(obj.gt) + length += 1 + len + } + if (defined(obj.gte)) { + var len = encodings.bytes.encodingLength(obj.gte) + length += 1 + len + } + if (defined(obj.lt)) { + var len = encodings.bytes.encodingLength(obj.lt) + length += 1 + len + } + if (defined(obj.lte)) { + var len = encodings.bytes.encodingLength(obj.lte) + length += 1 + len + } + if (defined(obj.limit)) { + var len = encodings.sint64.encodingLength(obj.limit) + length += 1 + len + } + if (defined(obj.reverse)) { + var len = encodings.bool.encodingLength(obj.reverse) + length += 1 + len + } + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (defined(obj.gt)) { + buf[offset++] = 26 + encodings.bytes.encode(obj.gt, buf, offset) + offset += encodings.bytes.encode.bytes + } + if (defined(obj.gte)) { + buf[offset++] = 34 + encodings.bytes.encode(obj.gte, buf, offset) + offset += encodings.bytes.encode.bytes + } + if (defined(obj.lt)) { + buf[offset++] = 42 + encodings.bytes.encode(obj.lt, buf, offset) + offset += encodings.bytes.encode.bytes + } + if (defined(obj.lte)) { + buf[offset++] = 50 + encodings.bytes.encode(obj.lte, buf, offset) + offset += encodings.bytes.encode.bytes + } + if (defined(obj.limit)) { + buf[offset++] = 56 + encodings.sint64.encode(obj.limit, buf, offset) + offset += encodings.sint64.encode.bytes + } + if (defined(obj.reverse)) { + buf[offset++] = 64 + encodings.bool.encode(obj.reverse, buf, offset) + offset += encodings.bool.encode.bytes + } + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + gt: null, + gte: null, + lt: null, + lte: null, + limit: 0, + reverse: false + } + while (true) { + if (end <= offset) { + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 3: + obj.gt = encodings.bytes.decode(buf, offset) + offset += encodings.bytes.decode.bytes + break + case 4: + obj.gte = encodings.bytes.decode(buf, offset) + offset += encodings.bytes.decode.bytes + break + case 5: + obj.lt = encodings.bytes.decode(buf, offset) + offset += encodings.bytes.decode.bytes + break + case 6: + obj.lte = encodings.bytes.decode(buf, offset) + offset += encodings.bytes.decode.bytes + break + case 7: + obj.limit = encodings.sint64.decode(buf, offset) + offset += encodings.sint64.decode.bytes + break + case 8: + obj.reverse = encodings.bool.decode(buf, offset) + offset += encodings.bool.decode.bytes + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } + } + + Clear.encodingLength = encodingLength + Clear.encode = encode + Clear.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.id)) throw new Error("id is required") + var len = encodings.varint.encodingLength(obj.id) + length += 1 + len + if (defined(obj.options)) { + var len = ClearOptions.encodingLength(obj.options) + length += varint.encodingLength(len) + length += 1 + len + } + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.id)) throw new Error("id is required") + buf[offset++] = 8 + encodings.varint.encode(obj.id, buf, offset) + offset += encodings.varint.encode.bytes + if (defined(obj.options)) { + buf[offset++] = 18 + varint.encode(ClearOptions.encodingLength(obj.options), buf, offset) + offset += varint.encode.bytes + ClearOptions.encode(obj.options, buf, offset) + offset += ClearOptions.encode.bytes + } + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + id: 0, + options: null + } + var found0 = false + while (true) { + if (end <= offset) { + if (!found0) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.id = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + found0 = true + break + case 2: + var len = varint.decode(buf, offset) + offset += varint.decode.bytes + obj.options = ClearOptions.decode(buf, offset, offset + len) + offset += ClearOptions.decode.bytes + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + +function defineIterator () { + var Options = Iterator.Options = { + buffer: true, + encodingLength: null, + encode: null, + decode: null + } + + defineOptions() + + function defineOptions () { + Options.encodingLength = encodingLength + Options.encode = encode + Options.decode = decode + function encodingLength (obj) { var length = 0 if (defined(obj.keys)) { @@ -515,8 +732,8 @@ function defineIterator () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -563,8 +780,8 @@ function defineIterator () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -625,13 +842,13 @@ function defineIterator () { offset = skip(prefix & 7, buf, offset) } } - } - } - - Iterator.encodingLength = encodingLength - Iterator.encode = encode - Iterator.decode = decode - + } + } + + Iterator.encodingLength = encodingLength + Iterator.encode = encode + Iterator.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -647,8 +864,8 @@ function defineIterator () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -671,8 +888,8 @@ function defineIterator () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -713,14 +930,14 @@ function defineIterator () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineCallback () { - Callback.encodingLength = encodingLength - Callback.encode = encode - Callback.decode = decode - + } +} + +function defineCallback () { + Callback.encodingLength = encodingLength + Callback.encode = encode + Callback.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -735,8 +952,8 @@ function defineCallback () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -757,8 +974,8 @@ function defineCallback () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -797,14 +1014,14 @@ function defineCallback () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineIteratorData () { - IteratorData.encodingLength = encodingLength - IteratorData.encode = encode - IteratorData.decode = decode - + } +} + +function defineIteratorData () { + IteratorData.encodingLength = encodingLength + IteratorData.encode = encode + IteratorData.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -823,8 +1040,8 @@ function defineIteratorData () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -850,8 +1067,8 @@ function defineIteratorData () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -895,9 +1112,9 @@ function defineIteratorData () { offset = skip(prefix & 7, buf, offset) } } - } -} - + } +} + function defined (val) { return val !== null && val !== undefined && (typeof val !== 'number' || !isNaN(val)) -} +} diff --git a/schema.proto b/schema.proto index dda1c99..f4b339a 100644 --- a/schema.proto +++ b/schema.proto @@ -25,6 +25,20 @@ message Batch { } } +message Clear { + required uint32 id = 1; + optional ClearOptions options = 2; + + message ClearOptions { + optional bytes gt = 3; + optional bytes gte = 4; + optional bytes lt = 5; + optional bytes lte = 6; + optional sint32 limit = 7; + optional bool reverse = 8; + } +} + message Iterator { required uint32 id = 1; optional uint32 batch = 2; diff --git a/server.js b/server.js index c0764f5..b942dd6 100644 --- a/server.js +++ b/server.js @@ -15,7 +15,8 @@ const DECODERS = [ messages.Put, messages.Delete, messages.Batch, - messages.Iterator + messages.Iterator, + messages.Clear ] module.exports = function (db, opts) { @@ -66,6 +67,7 @@ module.exports = function (db, opts) { case 2: return onreadonly(req) case 3: return onreadonly(req) case 4: return oniterator(req) + case 5: return onreadonly(req) } } else { switch (tag) { @@ -74,6 +76,7 @@ module.exports = function (db, opts) { case 2: return ondel(req) case 3: return onbatch(req) case 4: return oniterator(req) + case 5: return onclear(req) } } }) @@ -138,6 +141,12 @@ module.exports = function (db, opts) { prev.next() } } + + function onclear (req) { + down.clear(cleanRangeOptions(req.options), function (err) { + callback(req.id, err) + }) + } } }