CDB: add mmap backend (#183)

* CDB: add mmap backend

User now has the choice of running the CDB from heap or from an
mmapped file.

* fix indentation

* remove cruft from response.h

[ci skip]

src/protocol/data/memcache/response.h

@@ -105,3 +105,4 @@ void response_reset(struct response *rsp);
 struct response *response_borrow(void);
 void response_return(struct response **rsp);
 void response_return_all(struct response **rsp); /* return all responses in chain */
+

src/server/cdb/main.c

@@ -90,7 +90,8 @@ setup_cdb_handle(cdb_options_st *opt)
         log_stderr("cdb_file_path option not set, cannot continue");
         exit(EX_CONFIG);
     }
-    return cdb_handle_create(cdb_file_path);
+
+    return cdb_handle_create(cdb_file_path, CDB_MMAP);
 }
 
 static void

src/storage/cdb/cdb.h

@@ -6,14 +6,13 @@
 
 struct CDBHandle;
 
-struct CDBBString {
-    uint32_t len;   /* string length */
-    char     *data; /* string data */
+enum CDBStoreMethod {
+	CDB_HEAP = 1,
+	CDB_MMAP = 2,
 };
 
-struct CDBData;
+struct CDBHandle* cdb_handle_create(const char *path, enum CDBStoreMethod meth);
 
-struct CDBHandle* cdb_handle_create(const char *path);
 void cdb_handle_destroy(struct CDBHandle *h);
 
 void cdb_setup(void);

src/storage/cdb/cdb_ffi/src/ccommon/bstring.rs

@@ -1,10 +1,10 @@
 use cdb_ccommon::bindings as bind;
 
-use std::slice;
+use std::convert::AsMut;
 use std::io;
 use std::ops::{Deref, DerefMut};
 use std::os::raw::c_char;
-use std::convert::AsMut;
+use std::slice;
 
 /// BStringRef provides a wrapper around a raw pointer to a cc_bstring. It's important to note that
 /// this struct does not take ownership of the underlying pointer, nor does it free it when it's
@@ -19,7 +19,7 @@ impl<'a> BStringRef<'a> {
     pub unsafe fn from_raw(ptr: *const bind::bstring) -> Self {
         assert!(!ptr.is_null());
         let inner = &*ptr;
-        BStringRef{inner}
+        BStringRef { inner }
     }
 
     #[allow(dead_code)]
@@ -34,14 +34,13 @@ impl<'a> Deref for BStringRef<'a> {
     fn deref(&self) -> &<Self as Deref>::Target {
         unsafe {
             slice::from_raw_parts(
-                self.inner.data as *const c_char as *const u8,  // cast *const i8 -> *const u8
-                self.inner.len as usize
+                self.inner.data as *const c_char as *const u8, // cast *const i8 -> *const u8
+                self.inner.len as usize,
             )
         }
     }
 }
 
-
 #[derive(Debug)]
 pub struct BStringRefMut<'a> {
     inner: &'a mut bind::bstring,
@@ -50,7 +49,7 @@ pub struct BStringRefMut<'a> {
 impl<'a> BStringRefMut<'a> {
     pub unsafe fn from_raw(ptr: *mut bind::bstring) -> Self {
         assert!(!ptr.is_null());
-        BStringRefMut{inner: &mut *ptr}
+        BStringRefMut { inner: &mut *ptr }
     }
 
     pub fn into_raw(self) -> *mut bind::bstring {
@@ -65,20 +64,15 @@ impl<'a> Deref for BStringRefMut<'a> {
         unsafe {
             slice::from_raw_parts(
                 self.inner.data as *const c_char as *const u8,
-                self.inner.len as usize
+                self.inner.len as usize,
             )
         }
     }
 }
 
 impl<'a> DerefMut for BStringRefMut<'a> {
     fn deref_mut(&mut self) -> &mut <Self as Deref>::Target {
-        unsafe {
-            slice::from_raw_parts_mut(
-                self.inner.data as *mut u8,
-                self.inner.len as usize
-            )
-        }
+        unsafe { slice::from_raw_parts_mut(self.inner.data as *mut u8, self.inner.len as usize) }
     }
 }
 
@@ -98,19 +92,18 @@ impl<'a> AsMut<bind::bstring> for BStringRefMut<'a> {
     }
 }
 
-
 #[cfg(test)]
 mod test {
     use super::*;
-    use std::ffi::CString;
     use std::boxed::Box;
+    use std::ffi::CString;
     use std::io::Write;
 
     struct BStringStr<'a>(&'a str);
 
     impl<'a> BStringStr<'a> {
         fn into_raw(self) -> *mut bind::bstring {
-            let bs = bind::bstring{
+            let bs = bind::bstring {
                 len: self.0.len() as u32,
                 data: CString::new(self.0).unwrap().into_raw(),
             };
@@ -155,7 +148,6 @@ mod test {
             let n = buf.write(&d).unwrap();
             assert_eq!(n, 3);
         }
-
 
         assert_eq!(&bsr[0..3], &d[0..3]);
 

src/storage/cdb/cdb_ffi/src/lib.rs

@@ -1,4 +1,5 @@
-#[macro_use] extern crate log;
+#[macro_use]
+extern crate log;
 extern crate bytes;
 extern crate cdb_rs;
 extern crate env_logger;
@@ -11,63 +12,102 @@ mod ccommon;
 use std::convert::From;
 use std::ffi::CStr;
 use std::os::raw::c_char;
+use std::path::PathBuf;
 use std::ptr;
 
 use cdb_rs::cdb;
-use cdb_rs::{Result, CDB};
+use cdb_rs::{Mmap, Result, CDB};
 
-use cdb_ccommon::bindings as bind;
 use ccommon::bstring::{BStringRef, BStringRefMut};
+use cdb_ccommon::bindings as bind;
+
+#[repr(C)]
+pub enum CDBData {
+    Boxed(Box<[u8]>),
+    Mmapped(Mmap),
+}
 
 #[repr(C)]
 pub struct CDBHandle {
-    inner: Box<[u8]>,
+    data: CDBData,
+}
+
+#[repr(C)]
+pub enum CDBStoreMethod {
+    HEAP = 1,
+    MMAP = 2,
 }
 
 impl CDBHandle {
-    pub unsafe fn from_raw<'a>(ptr: *mut CDBHandle) -> &'a CDBHandle { &*ptr }
+    pub unsafe fn from_raw<'a>(ptr: *mut CDBHandle) -> &'a CDBHandle {
+        &*ptr
+    }
+
+    pub fn new(data: CDBData) -> CDBHandle {
+        CDBHandle { data }
+    }
 }
 
-impl<'a> From<&'a CDBHandle> for CDB<'a> {
-    fn from(h: &'a CDBHandle) -> Self {
-        CDB::new(&h.inner)
+impl AsRef<[u8]> for CDBHandle {
+    #[inline]
+    fn as_ref(&self) -> &[u8] {
+        match self.data {
+            CDBData::Boxed(ref bx) => bx.as_ref(),
+            CDBData::Mmapped(ref mm) => mm.as_ref(),
+        }
     }
 }
 
-fn mk_cdb_handler(path: String) -> Result<CDBHandle> {
-    assert!(
-        !path.is_empty(),
-        "cdb file path was empty, misconfiguration?"
-    );
-    debug!("mk_cdb_handler, path: {:?}", path);
-    let inner = cdb::load_bytes_at_path(path.as_ref())?;
-    debug!("inner: {:?}", inner);
+enum LoadOption {
+    Heap(PathBuf),
+    Mmap(PathBuf),
+}
 
-    Ok(CDBHandle { inner })
+#[inline]
+fn mk_cdb_handler(lo: LoadOption) -> Result<CDBHandle> {
+    match lo {
+        LoadOption::Heap(pb) => cdb::mmap_bytes_at_path(&pb)
+            .map(|mm| CDBHandle::new(CDBData::Mmapped(mm))),
+        LoadOption::Mmap(pb) => cdb::load_bytes_at_path(&pb)
+            .map(|b| CDBHandle::new(CDBData::Boxed(b))),
+    }
 }
 
-fn cstr_to_string(s: *const c_char) -> Result<String> {
+#[inline]
+fn cstr_to_path_buf(s: *const c_char) -> Result<PathBuf> {
     let ps = unsafe { CStr::from_ptr(s) }.to_str()?;
-    let rv = String::from(ps);
+
+    assert!(!ps.is_empty(), "cdb file path was empty, misconfiguration?");
+
+    let rv = PathBuf::from(ps);
     eprintln!("cstr_to_string: {:?}", rv);
 
     Ok(rv)
 }
 
 #[no_mangle]
-pub extern "C" fn cdb_handle_create(path: *const c_char) -> *mut CDBHandle {
+pub extern "C" fn cdb_handle_create(
+    path: *const c_char,
+    opt: CDBStoreMethod
+) -> *mut CDBHandle {
     assert!(!path.is_null());
 
-    match cstr_to_string(path).and_then(|s| mk_cdb_handler(s)) {
-        Ok(bhandle) => Box::into_raw(Box::new(bhandle)),
-        Err(err) => {
-            error!("failed to create CDBHandle: {:?}", err);
+    cstr_to_path_buf(path)
+        .and_then(|pathbuf| {
+            mk_cdb_handler(
+                match opt {
+                    CDBStoreMethod::HEAP => LoadOption::Heap(pathbuf),
+                    CDBStoreMethod::MMAP => LoadOption::Mmap(pathbuf),
+                }
+            )
+        })
+        .map(|h| Box::into_raw(Box::new(h)))
+        .unwrap_or_else(|err| {
+            error!("failed to create cdb_handle {:?}", err);
             ptr::null_mut()
-        }
-    }
+        })
 }
 
-
 #[no_mangle]
 pub extern "C" fn cdb_get(
     h: *mut CDBHandle,
@@ -83,7 +123,7 @@ pub extern "C" fn cdb_get(
     let key = unsafe { BStringRef::from_raw(k) };
     let mut val = unsafe { BStringRefMut::from_raw(v) };
 
-    match CDB::from(handle).get(&key, &mut val)  {
+    match CDB::new(handle).get(&key, &mut val) {
         Ok(Some(n)) => {
             {
                 // this provides access to the underlying struct fields
@@ -92,8 +132,8 @@ pub extern "C" fn cdb_get(
                 let mut vstr = val.as_mut();
                 vstr.len = n as u32;
             }
-            val.into_raw()  // consume BufStringRefMut and return the underlying raw pointer
-        },
+            val.into_raw() // consume BufStringRefMut and return the underlying raw pointer
+        }
         Ok(None) => ptr::null_mut(), // not found, return a NULL
         Err(err) => {
             eprintln!("got error: {:?}", err);
@@ -102,7 +142,6 @@ pub extern "C" fn cdb_get(
     }
 }
 
-
 #[no_mangle]
 pub extern "C" fn cdb_handle_destroy(handle: *mut CDBHandle) {
     unsafe {

src/storage/cdb/cdb_rs/src/cdb/mod.rs

@@ -6,6 +6,8 @@ use std::fmt;
 use std::fs::File;
 use std::io::Read;
 use std::result;
+use memmap::Mmap;
+use std::path::Path;
 
 pub mod errors;
 pub mod input;
@@ -127,19 +129,25 @@ struct IndexEntry {
 #[derive(Debug)]
 #[repr(C)]
 pub struct CDB<'a> {
-    data: &'a [u8],
+    data: &'a [u8]
 }
 
-pub fn load_bytes_at_path(path: &str) -> Result<Box<[u8]>> {
+
+pub fn load_bytes_at_path(path: &Path) -> Result<Box<[u8]>> {
     let mut f = File::open(path)?;
     let mut buffer = Vec::with_capacity(f.metadata()?.len() as usize);
     f.read_to_end(&mut buffer)?;
     Ok(buffer.into_boxed_slice())
 }
 
+pub fn mmap_bytes_at_path(path: &Path) -> Result<Mmap> {
+    let f = File::open(path)?;
+    unsafe { Mmap::map(&f) }.map_err(|e| e.into())
+}
+
 impl<'a> CDB<'a> {
-    pub fn new<'b>(b: &'b [u8]) -> CDB<'b> {
-        CDB { data: b }
+    pub fn new<'b, T: AsRef<[u8]>>(r: &'b T) -> CDB<'b> {
+        CDB { data: r.as_ref() }
     }
 
     #[inline]
@@ -215,7 +223,6 @@ impl<'a> CDB<'a> {
                 return Ok(None);
             } else if idx_ent.hash == hash {
                 let kv = self.get_kv_ref(idx_ent)?;
-                // TODO: this is incorrect handling of the buffer! shit!
                 if &kv.k[..] == key {
                     return Ok(Some(copy_slice(buf, kv.v)));
                 } else {
@@ -268,7 +275,7 @@ mod tests {
             }
         }).unwrap();
 
-        load_bytes_at_path(path.to_str().unwrap())
+        load_bytes_at_path(&path)
     }
 
 
@@ -290,7 +297,7 @@ mod tests {
         for (k, v) in kvs {
             let mut buf = Vec::new();
             buf.resize(10, 0u8);
-            
+
             let n = cdb.get(k.as_bytes(), &mut buf[..]).unwrap().unwrap();
             assert_eq!(n, v.len());
             assert_eq!(&buf[0..n], v.as_bytes())

src/storage/cdb/cdb_rs/src/lib.rs

@@ -11,3 +11,4 @@ extern crate clap;
 
 pub mod cdb;
 pub use cdb::{CDB, Result, CDBError};
+pub use memmap::Mmap;