refactoring CLI commands and documenting

README.md

@@ -57,29 +57,198 @@ These all match the parameters used with
 [`levelup`](https://github.com/rvagg/node-levelup). The default encoding
 for the database is set to `json`.
 
-## --start <key-pattern>
-Specify the start of the current range. You can also use `gt` or `gte`.
+## --get <key>
+Get a value
+```sh
+lev --get foo
+```
 
-## --end <key-pattern>
-Specify the end of the current range. You can also use `lt` and `lte`.
+## --put <key>
+Put a value
+```sh
+lev --put foo --value bar
+```
+
+## --del <key>
+Delete a value
+```sh
+lev --del foo
+```
+
+## --batch <operations>
+Put or delete several values, using [`levelup` batch syntax](https://github.com/Level/levelup#dbbatcharray-options-callback-array-form)
+```sh
+lev --batch '[
+{"type":"del","key":"father"},
+{"type":"put","key":"name","value":"Yuri Irsenovich Kim"},
+{"type":"put","key":"dob","value":"16 February 1941"},
+{"type":"put","key":"spouse","value":"Kim Young-sook"},
+{"type":"put","key":"occupation","value":"Clown"}
+]'
+```
+or from a file
+```sh
+# there should be one entry per line
+# either as valid JSON
+echo '[
+{"type":"del","key":"father"},
+{"type":"put","key":"name","value":"Yuri Irsenovich Kim"},
+{"type":"put","key":"dob","value":"16 February 1941"},
+{"type":"put","key":"spouse","value":"Kim Young-sook"},
+{"type":"put","key":"occupation","value":"Clown"}
+]' > ops.json
+# or as newline-delimited JSON
+echo '
+{"type":"del","key":"father"}
+{"type":"put","key":"name","value":"Yuri Irsenovich Kim"}
+{"type":"put","key":"dob","value":"16 February 1941"}
+{"type":"put","key":"spouse","value":"Kim Young-sook"}
+{"type":"put","key":"occupation","value":"Clown"}
+' > ops.json
+lev --batch ./ops.json
+```
+
+### Import / Export
+If the type is omitted, defaults to `put`, which allows to use the command to do imports/exports, in combination with [`--all`](#--all):
+```sh
+lev --all > leveldb.export
+lev /tmp/my-new-db --batch leveldb.export
+```
+If it's a large export, you can compress it on the fly
+```sh
+lev --all | gzip -9 > leveldb.export.gz
+gzip -dk leveldb.export.gz
+lev /tmp/my-new-db --batch leveldb.export
+```
+
+### Delete by range
+The `--batch` option can also be used to delete key/values by range in 2 steps:
+```
+# 1 - collect all the key/values to delete
+lev --all --start 'foo' --end 'fooz' > ./to_delete
+# 2 - pass the file as argument to the --batch option with a --del flag
+lev --batch ./to_delete --del
+```
+
+## --keys
+List all the keys in the current range. Will tabularize the output by default (see `--line`).
+```sh
+lev --keys
+```
 
 ## --values
-Only list the all of the values in the current range.
+List all the values in the current range.
 Emit as a new-line delimited stream of json.
+```sh
+lev --values
+```
 
-## --keys
-Only list all of the keys in the current range. Will tabularize the output.
+## --all
+List all the keys and values in the current range.
+Emit as a new-line delimited stream of json.
+```sh
+lev --all
+```
+It can be used to create an export of the database, to be imported with [`--batch`](#--batch)
+```sh
+lev --all > leveldb.export
+lev /tmp/my-new-db --batch leveldb.export
+```
 
-## --keyEncoding <string>
-Specify the encoding for the keys.
+## --start <key-pattern>
+Specify the start of the current range. You can also use `gt` or `gte`.
+```sh
+# output all keys after 'foo'
+lev --keys --start 'foo'
+# which is equivalent to
+lev --keys --gte 'foo'
+# the same for values
+lev --values --start 'foo'
+```
 
-## --valueEncoding <string>
-Specify the encoding for the values.
+## --end <key-pattern>
+Specify the end of the current range. You can also use `lt` and `lte`.
+```sh
+# output all keys before 'fooz'
+lev --keys --end 'fooz'
+# which is equivalent to
+lev --keys --lte 'fooz'
+# the same for values
+lev --values --end 'fooz'
+# output all keys between 'foo' and 'fooz'
+lev --keys --start 'foo' --end 'fooz'
+```
+
+## --match <key-pattern>
+Filter keys or values by a pattern applied on the key
+```sh
+lev  --keys --match 'f*'
+lev  --values --match 'f*'
+lev  --all --match 'f*'
+# Equivalent to
+lev --match 'f*'
+```
+
+See [`minimatch` doc](https://github.com/isaacs/minimatch#readme) for patterns
 
 ## --limit <number>
 Limit the number of records emitted in the current range.
+```sh
+lev --keys --limit 10
+lev --values --start 'foo' --end 'fooz' --limit 100
+lev --match 'f*' --limit 10
+```
 
 ## --reverse
 Reverse the stream.
+```sh
+lev --keys --reverse
+lev --keys --start 'foo' --end 'fooz' --limit 100 --reverse
+```
 
+## --line
+Output one key per line (instead of the default tabularized output)
+```sh
+lev --keys --line
+```
+
+## --length
+Output the length of the current range
+```sh
+# Count all the key/value pairs in the database
+lev --length
+# Counts the keys and values between 'foo' and 'fooz'
+lev --start 'foo' --end 'fooz' --length
+```
+
+## --valueEncoding <string>
+Specify the encoding for the values (Defaults to 'json').
+```sh
+lev --values --valueEncoding buffer
+```
+
+## --location <string>
+Specify the path to the LevelDB to use. Defaults to the current directory.
+```sh
+lev --location /tmp/test-db --keys
+# Equivalent to
+lev /tmp/test-db --keys
+```
+
+## --map <JS function string or path>
+Pass streams results in a map function
+* either inline
+```sh
+lev --keys --map 'key => key.split(":")[1]'
+lev --all --map 'data => data.value.replace(data.key, "")'
+```
+* or from a JS file that exports a function
+```js
+# in ./map_fn.js
+module.exports = key => key.split(":")[1]
+```
+```sh
+lev --keys --map ./map_fn.js
+```
 
+If the function, returns null or undefined, the result is filtered-out

index.js

@@ -12,22 +12,31 @@ module.exports = function(args) {
   // find where the location by examining the arguments
   // and create an instance to work with.
   //
-  locate(args);
+  locate(args, function (err) {
+    if (err) {
+      console.error(err);
+      return process.exit(1);
+    }
+    init(args);
+  });
+};
+
+function init (args) {
   var db = getDB(args);
 
   //
   // if any of these commands are specified as arguments
   // than the program should not be run in REPL mode.
   //
   var cliCommands = [
-    'keys', 'values', 'get', 'match',
-    'put', 'del', 'createReadStream', 'batch'
+    'keys', 'values', 'get', 'match', 'put', 'del',
+    'all', 'batch', 'length', 'start', 'end', 'limit', 'map'
   ];
-  
+
   var cliMode = Object.keys(args).some(function(cmd) {
     return cliCommands.indexOf(cmd) > -1;
   });
- 
+
   if (cliMode) {
     return cli(db, args);
   }
@@ -39,6 +48,5 @@ module.exports = function(args) {
 
   history(repl, args);
   completion(repl, cache);
-
 };
 

lib/batch_from_file.js

@@ -0,0 +1,52 @@
+var fs = require('fs');
+var path = require('path');
+var split = require('split');
+var printAndExit = require('./print_and_exit');
+
+module.exports = function(db, file, del) {
+  var batch = [];
+  var total = 0;
+
+  fs.createReadStream(path.resolve(file))
+  .pipe(split())
+  .on('data', function(line) {
+    if (line[0] !== '{') return
+    line = line.replace(/,$/, '')
+
+    //
+    // add each line to the batch
+    //
+    var entry = JSON.parse(line);
+    entry.type = del ? 'del' : (entry.type || 'put');
+    batch.push(entry)
+
+    //
+    // run operations by batches of 10000 entries
+    //
+    if (batch.length >= 10000) {
+      var stream = this;
+      stream.pause();
+      db.batch(batch, function(err, val) {
+        if (err) return printAndExit(err, val);
+        total += batch.length
+        console.log('ops run:', total)
+        batch = [];
+        stream.resume();
+      })
+    }
+
+  })
+  .on('close', function() {
+    if (batch.length > 0) {
+      db.batch(batch, function(err, val) {
+        if (err) return printAndExit(err, val);
+        total += batch.length;
+        console.log('total ops:', total)
+      });
+    }
+    else {
+      console.log('total ops:', total)
+    }
+  })
+}
+

lib/cache.js

@@ -1,6 +1,6 @@
 /*
  *
- * cahce.js
+ * cache.js
  * save the keys from the readstream into an
  * array so that they can be autocompleted and suggested.
  *