Add WASI support

.github/workflows/ci.yml

@@ -104,3 +104,12 @@ jobs:
           pip3 install --upgrade pip
           pip3 install maturin
           just test_py
+
+      - name: Run WASI tests
+        if: startsWith(matrix.os, 'macos')
+        run: |
+          rustup toolchain install nightly
+          rustup target add wasm32-wasi
+          brew install wasmtime
+          cargo install cargo-wasi
+          just test_wasi

Cargo.toml

@@ -22,13 +22,17 @@ libc = "0.2.104"
 log = {version = "0.4.17", optional = true }
 pyo3 = {version = "0.18.0", features=["extension-module", "abi3-py37"], optional = true }
 
+# Common test/bench dependencies
 [dev-dependencies]
+rand = "0.8"
+tempfile = "3.5.0"
+
+# Just benchmarking dependencies
+[target.'cfg(not(target_os = "wasi"))'.dev-dependencies]
 ctrlc = "3.2.3"
 fastrand = "1.9.0"
-rand = "0.8"
 lmdb-rkv = "0.14.0"
 sanakirja = "1.3.3"
-tempfile = "3.5.0"
 sled = "0.34.7"
 rocksdb = "0.21.0"
 libc = "0.2.99"

justfile

@@ -27,11 +27,14 @@ install_py: pre
 test: pre
     RUST_BACKTRACE=1 cargo test
 
+test_wasi: pre
+    CARGO_TARGET_WASM32_WASI_RUNNER="wasmtime --mapdir=/::$TMPDIR" cargo +nightly wasi test -- --nocapture
+
 bench bench='lmdb_benchmark': pre
     cargo bench --bench {{bench}}
 
 watch +args='test':
-  cargo watch --clear --exec "{{args}}"
+    cargo watch --clear --exec "{{args}}"
 
 fuzz: pre
     cargo fuzz run --sanitizer=none fuzz_redb -- -max_len=100000

src/db.rs

@@ -750,7 +750,7 @@ impl Builder {
     /// Set the amount of memory (in bytes) used for caching data
     pub fn set_cache_size(&mut self, bytes: usize) -> &mut Self {
         // TODO: allow dynamic expansion of the read/write cache
-        self.read_cache_size_bytes = bytes * 9 / 10;
+        self.read_cache_size_bytes = bytes / 10 * 9;
         self.write_cache_size_bytes = bytes / 10;
         self
     }
@@ -810,13 +810,11 @@ impl std::fmt::Debug for Database {
 
 #[cfg(test)]
 mod test {
-    use tempfile::NamedTempFile;
-
     use crate::{Database, Durability, Error, ReadableTable, TableDefinition};
 
     #[test]
     fn small_pages() {
-        let tmpfile: NamedTempFile = NamedTempFile::new().unwrap();
+        let tmpfile = crate::create_tempfile();
 
         let db = Database::builder()
             .set_page_size(512)
@@ -833,7 +831,7 @@ mod test {
 
     #[test]
     fn small_pages2() {
-        let tmpfile: NamedTempFile = NamedTempFile::new().unwrap();
+        let tmpfile = crate::create_tempfile();
 
         let db = Database::builder()
             .set_page_size(512)
@@ -927,7 +925,7 @@ mod test {
 
     #[test]
     fn small_pages3() {
-        let tmpfile: NamedTempFile = NamedTempFile::new().unwrap();
+        let tmpfile = crate::create_tempfile();
 
         let db = Database::builder()
             .set_page_size(1024)
@@ -960,7 +958,7 @@ mod test {
 
     #[test]
     fn small_pages4() {
-        let tmpfile: NamedTempFile = NamedTempFile::new().unwrap();
+        let tmpfile = crate::create_tempfile();
 
         let db = Database::builder()
             .set_cache_size(1024 * 1024)
@@ -997,7 +995,7 @@ mod test {
 
     #[test]
     fn crash_regression1() {
-        let tmpfile: NamedTempFile = NamedTempFile::new().unwrap();
+        let tmpfile = crate::create_tempfile();
 
         let db = Database::builder()
             .set_cache_size(1024 * 1024)
@@ -1020,7 +1018,7 @@ mod test {
 
     #[test]
     fn crash_regression2() {
-        let tmpfile: NamedTempFile = NamedTempFile::new().unwrap();
+        let tmpfile = crate::create_tempfile();
 
         let db = Database::builder()
             .set_cache_size(48101213 / 5)
@@ -1101,7 +1099,7 @@ mod test {
 
     #[test]
     fn dynamic_shrink() {
-        let tmpfile: NamedTempFile = NamedTempFile::new().unwrap();
+        let tmpfile = crate::create_tempfile();
         let table_definition: TableDefinition<u64, &[u8]> = TableDefinition::new("x");
         let big_value = vec![0u8; 1024];
 

src/lib.rs

@@ -6,6 +6,8 @@
     clippy::cast_sign_loss,
     clippy::disallowed_methods
 )]
+// TODO remove this once wasi no longer requires nightly
+#![cfg_attr(target_os = "wasi", feature(wasi_ext))]
 
 pub use db::{
     Builder, Database, MultimapTableDefinition, MultimapTableHandle, TableDefinition, TableHandle,
@@ -37,3 +39,12 @@ mod transactions;
 mod tree_store;
 mod tuple_types;
 mod types;
+
+#[cfg(test)]
+fn create_tempfile() -> tempfile::NamedTempFile {
+    if cfg!(target_os = "wasi") {
+        tempfile::NamedTempFile::new_in("/").unwrap()
+    } else {
+        tempfile::NamedTempFile::new().unwrap()
+    }
+}

src/table.rs

@@ -46,7 +46,7 @@ impl<'db, 'txn, K: RedbKey + 'static, V: RedbValue + 'static> Table<'db, 'txn, K
         let first = self
             .iter()?
             .next()
-            .map(|x| x.map(|(key, _)| K::as_bytes(key.value().borrow()).as_ref().to_vec()));
+            .map(|x| x.map(|(key, _)| K::as_bytes(&key.value()).as_ref().to_vec()));
         if let Some(owned_key) = first {
             let owned_key = owned_key?;
             let key = K::from_bytes(&owned_key);
@@ -65,7 +65,7 @@ impl<'db, 'txn, K: RedbKey + 'static, V: RedbValue + 'static> Table<'db, 'txn, K
             .iter()?
             .rev()
             .next()
-            .map(|x| x.map(|(key, _)| K::as_bytes(key.value().borrow()).as_ref().to_vec()));
+            .map(|x| x.map(|(key, _)| K::as_bytes(&key.value()).as_ref().to_vec()));
         if let Some(owned_key) = last {
             let owned_key = owned_key?;
             let key = K::from_bytes(&owned_key);

src/transactions.rs

@@ -1078,13 +1078,12 @@ impl<'a> Drop for ReadTransaction<'a> {
 #[cfg(test)]
 mod test {
     use crate::{Database, TableDefinition};
-    use tempfile::NamedTempFile;
 
     const X: TableDefinition<&str, &str> = TableDefinition::new("x");
 
     #[test]
     fn transaction_id_persistence() {
-        let tmpfile: NamedTempFile = NamedTempFile::new().unwrap();
+        let tmpfile = crate::create_tempfile();
         let db = Database::create(tmpfile.path()).unwrap();
         let write_txn = db.begin_write().unwrap();
         {

src/tree_store/btree.rs

@@ -267,7 +267,7 @@ impl<'a, K: RedbKey + 'a, V: RedbValue + 'a> BtreeMut<'a, K, V> {
             MutateHelper::new(&mut root, FreePolicy::Never, self.mem, &mut free_on_drop);
         for entry in iter {
             // TODO: optimize so that we don't have to call safe_delete in a loop
-            assert!(operation.safe_delete(entry?.key().borrow())?.is_some());
+            assert!(operation.safe_delete(&entry?.key())?.is_some());
         }
 
         let result = BtreeDrain::new(
@@ -304,7 +304,7 @@ impl<'a, K: RedbKey + 'a, V: RedbValue + 'a> BtreeMut<'a, K, V> {
             // TODO: optimize so that we don't have to call safe_delete in a loop
             let entry = entry?;
             if predicate(entry.key(), entry.value()) {
-                assert!(operation.safe_delete(entry.key().borrow())?.is_some());
+                assert!(operation.safe_delete(&entry.key())?.is_some());
             }
         }
 

src/tree_store/page_store/file_lock/mod.rs

@@ -1,6 +1,6 @@
-#[cfg(unix)]
+#[cfg(any(unix, target_os = "wasi"))]
 mod unix;
-#[cfg(unix)]
+#[cfg(any(unix, target_os = "wasi"))]
 pub(super) use unix::LockedFile;
 
 #[cfg(windows)]

src/tree_store/page_store/file_lock/unix.rs

@@ -1,14 +1,33 @@
+// TODO once Rust's libc has flock implemented for WASI, this file needs to be revisited.
+// What needs to be changed is commented below.
+// See also: https://github.com/WebAssembly/wasi-filesystem/issues/2
+
+// Remove this line once wasi-libc has flock
+#![cfg_attr(target_os = "wasi", allow(unused_imports))]
+
 use crate::{Error, Result};
 use std::fs::File;
 use std::io;
-use std::os::unix::fs::FileExt;
-use std::os::unix::io::AsRawFd;
+
+#[cfg(unix)]
+use std::os::unix::{fs::FileExt, io::AsRawFd};
+
+#[cfg(target_os = "wasi")]
+use std::os::wasi::{fs::FileExt, io::AsRawFd};
 
 pub(crate) struct LockedFile {
     file: File,
 }
 
 impl LockedFile {
+    // This is a no-op until we get flock in wasi-libc.
+    // Delete this function when we get flock.
+    #[cfg(target_os = "wasi")]
+    pub(crate) fn new(file: File) -> Result<Self> {
+        Ok(Self { file })
+    }
+
+    #[cfg(unix)] // remove this line when wasi-libc gets flock
     pub(crate) fn new(file: File) -> Result<Self> {
         let fd = file.as_raw_fd();
         let result = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) };
@@ -41,6 +60,7 @@ impl LockedFile {
     }
 }
 
+#[cfg(unix)] // remove this line when wasi-libc gets flock
 impl Drop for LockedFile {
     fn drop(&mut self) {
         unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) };

src/tree_store/page_store/header.rs

@@ -399,13 +399,12 @@ mod test {
     use std::fs::OpenOptions;
     use std::io::{Read, Seek, SeekFrom, Write};
     use std::mem::size_of;
-    use tempfile::NamedTempFile;
 
     const X: TableDefinition<&str, &str> = TableDefinition::new("x");
 
     #[test]
     fn repair_allocator_checksums() {
-        let tmpfile: NamedTempFile = NamedTempFile::new().unwrap();
+        let tmpfile = crate::create_tempfile();
         let db = Database::builder().create(tmpfile.path()).unwrap();
         let write_txn = db.begin_write().unwrap();
         {
@@ -517,7 +516,7 @@ mod test {
 
     #[test]
     fn repair_empty() {
-        let tmpfile: NamedTempFile = NamedTempFile::new().unwrap();
+        let tmpfile = crate::create_tempfile();
         let db = Database::builder().create(tmpfile.path()).unwrap();
         drop(db);
 
@@ -544,7 +543,7 @@ mod test {
 
     #[test]
     fn repair_insert_reserve_regression() {
-        let tmpfile: NamedTempFile = NamedTempFile::new().unwrap();
+        let tmpfile = crate::create_tempfile();
         let db = Database::builder().create(tmpfile.path()).unwrap();
 
         let def: TableDefinition<&str, &[u8]> = TableDefinition::new("x");

src/tree_store/page_store/page_manager.rs

@@ -821,7 +821,7 @@ impl TransactionalMemory {
         // Trim surplus file space, before finalizing the commit
         let shrunk = self.try_shrink(&mut state, &mut layout)?;
 
-        let mut secondary = state.header.secondary_slot_mut();
+        let secondary = state.header.secondary_slot_mut();
         secondary.transaction_id = transaction_id;
         secondary.user_root = data_root;
         secondary.system_root = system_root;
@@ -877,7 +877,7 @@ impl TransactionalMemory {
 
         let mut state = self.state.lock().unwrap();
         let layout = self.layout.lock().unwrap();
-        let mut secondary = state.header.secondary_slot_mut();
+        let secondary = state.header.secondary_slot_mut();
         secondary.transaction_id = transaction_id;
         secondary.user_root = data_root;
         secondary.system_root = system_root;