MRG: support check --upgrade to upgrade old versions of RocksDB/Rev…

…Index (#581)

* MRG: support multiple input files for singlesketch

* MRG: open rocksdb in rw mode so that old databases can be upgraded

* support check --upgrade; revert rw opening by default

* test 0.9.13 rocksdb too

* format

* cleanup

* fix

* add test to help debug

* add log?

* print out sourmash version

* print out sourmash version

* MRG: update to require sourmash v4.8.13

* upd

.gitignore

@@ -80,3 +80,7 @@ docs/_build/
 
 # pixi
 .pixi/
+
+*.csv
+*.zip
+*.rocksdb

src/check.rs

@@ -3,13 +3,21 @@ use anyhow::Result;
 
 use sourmash::index::revindex::{RevIndex, RevIndexOps};
 
-pub fn check(index: camino::Utf8PathBuf, quick: bool) -> Result<()> {
+pub fn check(index: camino::Utf8PathBuf, quick: bool, rw: bool) -> Result<()> {
     if !is_revindex_database(&index) {
         bail!("'{}' is not a valid RevIndex database", index);
     }
 
-    println!("Opening DB");
-    let db = RevIndex::open(index, true, None)?;
+    println!("Opening DB (rw mode? {})", rw);
+    let db = match RevIndex::open(index, !rw, None) {
+        Ok(db) => db,
+        Err(e) => {
+            return Err(anyhow::anyhow!(
+                "cannot open RocksDB database. Error is: {}",
+                e
+            ))
+        }
+    };
 
     println!("Starting check");
     db.check(quick);

src/fastmultigather_rocksdb.rs

@@ -28,7 +28,15 @@ pub fn fastmultigather_rocksdb(
         bail!("'{}' is not a valid RevIndex database", index);
     }
     // Open database once
-    let db = RevIndex::open(index, true, None)?;
+    let db = match RevIndex::open(index, true, None) {
+        Ok(db) => db,
+        Err(e) => {
+            return Err(anyhow::anyhow!(
+                "cannot open RocksDB database. Error is: {}",
+                e
+            ))
+        }
+    };
     println!("Loaded DB");
 
     // grab scaled from the database.

src/index.rs

@@ -69,7 +69,10 @@ pub(crate) fn index_obj<P: AsRef<Path>>(
             let mut index = RevIndex::create(output.as_ref(), collection, use_colors)?;
 
             if use_internal_storage {
+                eprintln!("Internalizing storage.");
                 index.internalize_storage()?;
+            } else {
+                eprintln!("Using external storage - not copying sketches.");
             }
             Ok(())
         }

src/lib.rs

@@ -222,9 +222,9 @@ fn do_index(
 }
 
 #[pyfunction]
-fn do_check(index: String, quick: bool) -> anyhow::Result<u8> {
+fn do_check(index: String, quick: bool, rw: bool) -> anyhow::Result<u8> {
     let idx: PathBuf = index.into();
-    match check::check(idx, quick) {
+    match check::check(idx, quick, rw) {
         Ok(_) => Ok(0),
         Err(e) => {
             eprintln!("Error: {e}");

src/manysearch_rocksdb.rs

@@ -1,7 +1,6 @@
 /// manysearch_rocksdb: rocksdb-indexed version of manysearch.
 use anyhow::Result;
 use camino::Utf8PathBuf as PathBuf;
-use log::debug;
 use rayon::prelude::*;
 use std::sync::atomic;
 use std::sync::atomic::AtomicUsize;
@@ -30,9 +29,17 @@ pub fn manysearch_rocksdb(
     if !is_revindex_database(&index) {
         bail!("'{}' is not a valid RevIndex database", index);
     }
+
     // Open database once
-    debug!("Opened revindex: '{index}')");
-    let db = RevIndex::open(index, true, None)?;
+    let db = match RevIndex::open(index, true, None) {
+        Ok(db) => db,
+        Err(e) => {
+            return Err(anyhow::anyhow!(
+                "cannot open RocksDB database. Error is: {}",
+                e
+            ))
+        }
+    };
 
     println!("Loaded DB");
 

src/python/sourmash_plugin_branchwater/__init__.py

@@ -405,11 +405,19 @@ def __init__(self, p):
         super().__init__(p)
         p.add_argument("index", help="RocksDB index file created with 'index'")
         p.add_argument("--quick", action="store_true")
+        p.add_argument(
+            "--writable",
+            "--upgrade",
+            action="store_true",
+            help="open database in read-write mode to upgrade the internal format if needed",
+        )
 
     def main(self, args):
         notify(f"checking index '{args.index}'")
         super().main(args)
-        status = sourmash_plugin_branchwater.do_check(args.index, args.quick)
+        status = sourmash_plugin_branchwater.do_check(
+            args.index, args.quick, args.writable
+        )
         if status == 0:
             notify(f"...index is ok!")
         return status

src/python/tests/test-data/rocksdb/podar-ref-subset.branch0_9_13.external.rocksdb.README.txt

@@ -0,0 +1,8 @@
+Created with:
+
+sourmash scripts index podar-ref-subset.sig.zip -o podar-ref-subset.branch0_9_13.external.rocksdb -s 100_000 -k 31 --no-internal-storage
+
+using sourmash_plugin_branchwater v0.9.13.
+
+This command uses external storage for the sketches, so the zip file must be
+in the cwd in order to do a fastmultigather.

src/python/tests/test-data/rocksdb/podar-ref-subset.branch0_9_13.external.rocksdb/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000005

src/python/tests/test-data/rocksdb/podar-ref-subset.branch0_9_13.external.rocksdb/IDENTITY

@@ -0,0 +1 @@
+79684d80-0690-40da-9060-a5f52f3e0bfc