feat: package benchmark runner into a re-usable binary (canbench) (#175)

Packages the benchmarking logic into a re-usable binary which I'm
calling `canbench`.

Once this binary is generally available, canister developers can use
this tool by:
1. Installing `canbench` - something like `cargo install canbench`.
2. Define a `canbench.yml` configuration.
3. Run benchmarks by running the `canbench` command.

A few more improvements in the UX will be done first before `canbench`
is ready for general availability.

Author: ielashi

.gitignore

@@ -1,5 +1,6 @@
 **/target
 **/.dfx
+**/.canbench
 
 # Various IDEs and Editors
 .vscode/

Cargo.lock

@@ -34,11 +34,6 @@ serde_yaml = "0.9"
 tempfile = "3.3.0"
 test-strategy = "0.3.1"
 
-[[bench]]
-name = "benchmarks"
-harness = false
-path = "benchmarks/benchmark.rs"
-
 [workspace]
 members = ["benchmark-canisters"]
 

Cargo.toml

@@ -1,7 +1,10 @@
 #!/usr/bin/env bash
 set -Eexuo pipefail
 
-BENCH_OUTPUT=$(cargo bench)
+# Install cargobench
+cargo install --path ./profiler --features bin
+
+BENCH_OUTPUT=$(canbench)
 
 set +e
 REGRESSIONS=$( echo "$BENCH_OUTPUT" |  grep -c "regressed by" )

benchmarks/benchmark_ci.sh

@@ -5,12 +5,22 @@ edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[lib]
+name = "profiler"
+path = "src/lib.rs"
+
+[[bin]]
+name = "canbench"
+path = "src/bin.rs"
+required-features = ["bin"]
+
 [dependencies]
 candid.workspace = true
 ic-cdk.workspace= true
 maplit = "1.0.2"
 serde = "1.0"
 
+clap = { version = "4.0.11", features = ["derive"], optional = true }
 colored = { version = "2.0.4", optional = true }
 flate2 = { version = "1.0.28", optional = true }
 hex = { version = "0.4.3", optional = true }
@@ -31,3 +41,5 @@ benchmark = [
   "tempfile",
   "wasmparser",
 ]
+
+bin = ["clap", "benchmark"]

bench.yml renamed to canbench.yml

@@ -109,10 +109,16 @@ pub fn run_benchmarks(
     }
 }
 
-fn drun_path() -> PathBuf {
+// Path to the canbench directory where we keep internal data.
+fn canbench_dir() -> PathBuf {
     PathBuf::new()
-        .join(env::var("CARGO_MANIFEST_DIR").unwrap())
-        .join("drun")
+        .join(env::current_dir().unwrap())
+        .join(".canbench")
+}
+
+// Path to drun.
+fn drun_path() -> PathBuf {
+    canbench_dir().join("drun")
 }
 
 // Downloads drun if it's not already downloaded.
@@ -140,6 +146,9 @@ fn maybe_download_drun() {
 fn download_drun() {
     println!("Downloading drun (will be cached for future uses)...");
 
+    // Create the canbench directory if it doesn't exist.
+    std::fs::create_dir_all(canbench_dir()).unwrap();
+
     let os = if cfg!(target_os = "linux") {
         "linux"
     } else if cfg!(target_os = "macos") {

profiler/Cargo.toml

@@ -3,6 +3,8 @@
 use clap::Parser;
 use std::{collections::BTreeMap, fs::File, io::Read, path::PathBuf, process::Command};
 
+const CFG_FILE_NAME: &str = "canbench.yml";
+
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
 struct Args {
@@ -21,8 +23,21 @@ struct Args {
 fn main() {
     let args = Args::parse();
 
-    // Read and parse the configuration in `bench.yml` file.
-    let mut file = File::open("bench.yml").expect("bench.yml not found");
+    // Read and parse the configuration file.
+    let mut file = match File::open(CFG_FILE_NAME) {
+        Ok(file) => file,
+        Err(err) => {
+            match err.kind() {
+                std::io::ErrorKind::NotFound => {
+                    println!("{} not found in current directory.", CFG_FILE_NAME)
+                }
+                other => println!("Error while opening `{}`: {}", CFG_FILE_NAME, other),
+            }
+
+            std::process::exit(1);
+        }
+    };
+
     let mut config_str = String::new();
     file.read_to_string(&mut config_str).unwrap();
     let cfg: BTreeMap<String, String> = serde_yaml::from_str(&config_str).unwrap();