Merge pull request #143 from Clikengo/async

async/await: complete rewrite of foundationdb

Author: Vincent Rouillé

.travis.yml

@@ -27,20 +27,21 @@ matrix:
     - rust: nightly
       env: RUST_BACKTRACE=full
 
-    - rust: nightly
+    - rust: stable
       env: NAME=clippy
            RUST_BACKTRACE=full
            CLIPPY=true
       script:
-        - cargo clippy --all --all-features
+        - rustup component add clippy
+        - cargo clippy
 
-    - rust: nightly
+    - rust: stable
       env: NAME=rustfmt
            RUST_BACKTRACE=full
            RUSTFMT=true
       script:
-        - rustup component add rustfmt-preview
-        - cargo fmt --all -- --write-mode=diff
+        - rustup component add rustfmt
+        - cargo fmt --all -- --check
 
     ## bindingtester
     - rust: stable
@@ -56,13 +57,12 @@ before_install:
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then scripts/install_foundationdb_linux.sh ; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then scripts/install_foundationdb_macos.sh ; fi
   - if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then scripts/install_foundationdb_win64.sh ; export PATH="${PATH}:/c/Program Files/foundationdb/bin" ; fi
-  - if [[ "$CLIPPY" == "true" ]]; then cargo install clippy --force ; fi
 
 script:
   - cargo test --manifest-path foundationdb-sys/Cargo.toml
   - cargo test --manifest-path foundationdb-gen/Cargo.toml
   - cargo test --manifest-path foundationdb/Cargo.toml
-  - cargo test --manifest-path foundationdb/Cargo.toml --no-default-features --features fdb-6_0
+  - cargo test --manifest-path foundationdb/Cargo.toml --no-default-features --features fdb-6_0 --tests
   - cargo test --manifest-path foundationdb-bench/Cargo.toml
   - cargo run --manifest-path foundationdb/Cargo.toml --example class-scheduling
 

Cargo.toml

@@ -1,8 +1,11 @@
 [workspace]
-members = ["foundationdb", 
-           "foundationdb-sys",
-           "foundationdb-gen",
-           "foundationdb-bench"]
+members = [
+    "foundationdb", 
+    "foundationdb-sys",
+    "foundationdb-gen",
+    "foundationdb-bench",
+    "foundationdb-bindingtester",
+]
 
 [profile.release]
 debug = true

README.md

@@ -9,10 +9,13 @@ The repo consists of multiple crates
 
 | Library | Status | Description |
 |---------|--------|-------------|
-| **foundationdb** | [![](http://meritbadge.herokuapp.com/foundationdb)](https://crates.io/crates/foundationdb) [![foundationdb](https://docs.rs/foundationdb/badge.svg)](https://docs.rs/foundationdb) | High level FoundationDB client API |
-| **foundationdb-sys** | [![](http://meritbadge.herokuapp.com/foundationdb-sys)](https://crates.io/crates/foundationdb-sys) [![foundationdb-sys](https://docs.rs/foundationdb-sys/badge.svg)](https://docs.rs/foundationdb-sys) | C API bindings for FoundationDB |
+| [**foundationdb**](foundationdb/README.md) | [![](http://meritbadge.herokuapp.com/foundationdb)](https://crates.io/crates/foundationdb) [![foundationdb](https://docs.rs/foundationdb/badge.svg)](https://docs.rs/foundationdb) | High level FoundationDB client API |
+| [**foundationdb-sys**](foundationdb-sys/README.md) | [![](http://meritbadge.herokuapp.com/foundationdb-sys)](https://crates.io/crates/foundationdb-sys) [![foundationdb-sys](https://docs.rs/foundationdb-sys/badge.svg)](https://docs.rs/foundationdb-sys) | C API bindings for FoundationDB |
 | **foundationdb-gen** | n/a | Code generator for common options and types of FoundationDB |
 
+The current version requires rustc 1.39+ to work (async/await feature).
+The previous version (0.3) is still maintained and is available within the 0.3 branch.
+
 ## License
 
 Licensed under either of

appveyor.yml

@@ -1,4 +1,4 @@
-os: Visual Studio 2017
+os: Visual Studio 2019
 
 environment:
   matrix:
@@ -23,6 +23,6 @@ build_script:
   - cargo test --manifest-path foundationdb-sys/Cargo.toml
   - cargo test --manifest-path foundationdb-gen/Cargo.toml
   - cargo test --manifest-path foundationdb/Cargo.toml
-  - cargo test --manifest-path foundationdb/Cargo.toml --no-default-features --features fdb-6_0
+  - cargo test --manifest-path foundationdb/Cargo.toml --no-default-features --features fdb-6_0 --tests
   - cargo test --manifest-path foundationdb-bench/Cargo.toml
   - cargo run --manifest-path foundationdb/Cargo.toml --example class-scheduling

foundationdb-bench/Cargo.toml

@@ -1,7 +1,11 @@
 [package]
 name = "foundationdb-bench"
-version = "0.1.0"
-authors = ["Benjamin Fry <benjaminfry@me.com>"]
+version = "0.4.0"
+authors = [
+    "Benjamin Fry <benjaminfry@me.com>", 
+    "Vincent Rouillé <vincent@clikengo.com>",
+]
+edition = "2018"
 
 description = """
 Bindings to the C api for FoundationDB
@@ -20,12 +24,12 @@ travis-ci = { repository = "bluejekyll/foundationdb-rs" }
 
 
 [dependencies]
-rand = "0.7"
-futures = "0.1"
+env_logger = "0.7.1"
 foundationdb = { version = "*", path = "../foundationdb" }
-stopwatch = "0"
-log = "0.4"
-env_logger = "0.6"
-structopt = "0.2"
+futures = "0.3.0"
+log = "0.4.8"
+rand = "0.7.2"
+stopwatch = "0.0.7"
+structopt = "0.3.3"
 
 [build-dependencies]

foundationdb-bench/src/bin/fdb-bench.rs

@@ -16,8 +16,7 @@ use rand::rngs::mock::StepRng;
 use stopwatch::Stopwatch;
 use structopt::StructOpt;
 
-use fdb::error::*;
-use fdb::*;
+use crate::fdb::*;
 
 #[derive(Clone)]
 struct Counter {
@@ -38,72 +37,9 @@ impl Counter {
     }
 }
 
-struct BenchRunner {
-    #[allow(unused)]
-    db: Database,
-    counter: Counter,
-    key_buf: Vec<u8>,
-    val_buf: Vec<u8>,
-    rng: StepRng,
-    trx: Option<Transaction>,
-    trx_batch_size: usize,
-}
-
-impl BenchRunner {
-    fn new(db: Database, rng: StepRng, counter: Counter, opt: &Opt) -> Self {
-        let mut key_buf = Vec::with_capacity(opt.key_len);
-        key_buf.resize(opt.key_len, 0u8);
-
-        let mut val_buf = Vec::with_capacity(opt.val_len);
-        val_buf.resize(opt.val_len, 0u8);
-
-        let trx = db.create_trx().expect("failed to create trx");
-
-        Self {
-            db,
-            counter,
-            key_buf,
-            val_buf,
-
-            rng,
-            trx: Some(trx),
-            trx_batch_size: opt.trx_batch_size,
-        }
-    }
-
-    //TODO: impl future
-    fn run(self) -> Box<Future<Item = (), Error = Error>> {
-        Box::new(loop_fn(self, Self::step))
-    }
-
-    //TODO: impl future
-    fn step(mut self) -> Box<Future<Item = Loop<(), Self>, Error = Error>> {
-        let trx = self.trx.take().unwrap();
-
-        for _ in 0..self.trx_batch_size {
-            self.rng.fill_bytes(&mut self.key_buf);
-            self.rng.fill_bytes(&mut self.val_buf);
-            self.key_buf[0] = 0x01;
-            trx.set(&self.key_buf, &self.val_buf);
-        }
-
-        let f = trx.commit().map(move |trx| {
-            trx.reset();
-            self.trx = Some(trx);
-
-            if self.counter.decr(self.trx_batch_size) {
-                Loop::Continue(self)
-            } else {
-                Loop::Break(())
-            }
-        });
-        Box::new(f)
-    }
-}
-
 #[derive(Clone)]
 struct Bench {
-    db: Database,
+    db: Arc<Database>,
     opt: Opt,
 }
 
@@ -124,7 +60,9 @@ impl Bench {
             let range = start..end;
             let counter = counter.clone();
             let b = self.clone();
-            let handle = std::thread::spawn(move || b.run_range(range, counter).wait());
+            let handle = std::thread::spawn(move || {
+                futures::executor::block_on(b.clone().run_range(range, counter))
+            });
             handles.push(handle);
 
             start = end;
@@ -146,22 +84,39 @@ impl Bench {
         );
     }
 
-    fn run_range(
-        &self,
-        r: std::ops::Range<usize>,
-        counter: Counter,
-    ) -> Box<Future<Item = (), Error = Error>> {
-        let runners = r
-            .into_iter()
-            .map(|n| {
-                // With deterministic Rng, benchmark with same parameters will overwrite same set
-                // of keys again, which makes benchmark result stable.
-                let rng = StepRng::new(n as u64, 1);
-                BenchRunner::new(self.db.clone(), rng, counter.clone(), &self.opt).run()
-            }).collect::<Vec<_>>();
-
-        let f = join_all(runners).map(|_| ());
-        Box::new(f)
+    async fn run_range(&self, r: std::ops::Range<usize>, counter: Counter) -> FdbResult<()> {
+        try_join_all(r.map(|n| {
+            // With deterministic Rng, benchmark with same parameters will overwrite same set
+            // of keys again, which makes benchmark result stable.
+            let rng = StepRng::new(n as u64, 1);
+            self.run_bench(rng, counter.clone())
+        }))
+        .await?;
+        Ok(())
+    }
+
+    async fn run_bench(&self, mut rng: StepRng, counter: Counter) -> FdbResult<()> {
+        let mut key_buf = vec![0; self.opt.key_len];
+
+        let mut val_buf = vec![0; self.opt.val_len];
+
+        let trx_batch_size = self.opt.trx_batch_size;
+        let mut trx = self.db.create_trx()?;
+
+        loop {
+            for _ in 0..trx_batch_size {
+                rng.fill_bytes(&mut key_buf);
+                rng.fill_bytes(&mut val_buf);
+                key_buf[0] = 0x01;
+                trx.set(&key_buf, &val_buf);
+            }
+
+            trx = trx.commit().await?.reset();
+
+            if !counter.decr(trx_batch_size) {
+                break Ok(());
+            }
+        }
     }
 }
 
@@ -189,34 +144,17 @@ struct Opt {
 fn main() {
     env_logger::init();
     let opt = Opt::from_args();
-
     info!("opt: {:?}", opt);
 
-    let network = fdb::init().expect("failed to init network");
-
-    let handle = std::thread::spawn(move || {
-        let error = network.run();
-
-        if let Err(error) = error {
-            panic!("fdb_run_network: {}", error);
-        }
-    });
-
-    network.wait();
-
-    let cluster_path = fdb::default_config_path();
-    let cluster = Cluster::new(cluster_path)
-        .wait()
-        .expect("failed to create cluster");
+    let network = fdb::boot().expect("failed to init network");
 
-    let db = cluster
-        .create_database()
-        .wait()
-        .expect("failed to get database");
+    let db = Arc::new(
+        futures::executor::block_on(fdb::Database::new_compat(None))
+            .expect("failed to get database"),
+    );
 
     let bench = Bench { db, opt };
     bench.run();
 
-    network.stop().expect("failed to stop network");
-    handle.join().expect("failed to join fdb thread");
+    drop(network);
 }

foundationdb-bindingtester/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "bindingtester"
+version = "0.4.0"
+authors = [
+    "Benjamin Fry <benjaminfry@me.com>", 
+    "Vincent Rouillé <vincent@clikengo.com>",
+]
+edition = "2018"
+
+description = """
+Bindings to the C api for FoundationDB
+"""
+
+repository = "https://github.com/bluejekyll/foundationdb-rs"
+
+readme = "README.md"
+keywords = ["foundationdb", "kv"]
+categories = ["database"]
+
+license = "MIT/Apache-2.0"
+
+[badges]
+travis-ci = { repository = "bluejekyll/foundationdb-rs" }
+
+[dependencies]
+env_logger = "0.7.1"
+foundationdb = { path = "../foundationdb", features = ["uuid"] }
+foundationdb-sys = { version = "0.4.0", path = "../foundationdb-sys", default-features = false }
+futures = "0.3.0"
+log = "0.4.8"
+structopt = "0.3.3"

foundationdb-bindingtester/README.md

@@ -0,0 +1,14 @@
+Rust FoundationDB bindingtester
+===============================
+
+This exe implements the official FoundationDB bindingtester protocol.
+
+By running `./bindingtester.py ${this_executable}`, you can test how the rust foundationdb bindings behave.
+
+The following configurations are tested and should pass without any issue:
+
+```
+./bindingtester.py --test-name scripted
+./bindingtester.py --num-ops 1000 --test-name api --api-version 610
+./bindingtester.py --num-ops 1000 --concurrency 5 --test-name api --api-version 610
+```