Add selective benchmarking and stores the benchmark in a CSV file

Author: hdvanegasm

test_suite/Cargo.lock

@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 anyhow = "1.0.89"
 clap = { version = "4.5.17", features = ["derive"] }
+csv = "1.3.0"
 env_logger = "0.11.5"
 itertools = "0.13.0"
 log = "0.4.22"

test_suite/Cargo.toml

@@ -0,0 +1,3 @@
+acir_opcodes,circuit_size,regex,with_gen_substr
+114,17865,email was meant for @[a-z]+,true
+59,7808,1=(a|b) (2=(b|c)+ )+d,true

test_suite/bench_result.csv

@@ -1,10 +1,10 @@
 {
+  "bench_all": false,
   "database": [
     {
       "regex": {
         "raw": "m(a|b)+-(c|d)+e"
       },
-      "with_bench": true,
       "input_size": 16,
       "samples_pass": [
         "mabab-cdcde",

test_suite/regex_db.json

@@ -1,7 +1,7 @@
-use std::{fmt::Display, process::Command};
+use std::{fmt::Display, path::Path, process::Command};
 
 use anyhow::Context;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
 use crate::constants;
@@ -16,14 +16,21 @@ pub enum Error {
 /// Results of the benchmark.
 ///
 /// This results are extracted using the command `bb gates -b <target>`
-#[derive(Deserialize)]
+#[derive(Deserialize, Serialize)]
 pub struct BenchResult {
     /// Number of ACIR opcodes generated by the compiler.
     pub acir_opcodes: u32,
     /// The number of gates.
     pub circuit_size: u32,
     /// Number of gates per opcode.
+    #[serde(skip_serializing)]
     pub gates_per_opcode: Vec<u32>,
+    /// Regex
+    #[serde(skip_deserializing)]
+    pub regex: String,
+    /// Tells if this benchmark was performed using the gen_substr() function.
+    #[serde(skip_deserializing)]
+    pub with_gen_substr: bool,
 }
 
 impl Display for BenchResult {
@@ -36,6 +43,31 @@ impl Display for BenchResult {
     }
 }
 
+/// Container for the benchmark results for each test.
+#[derive(Serialize, Default)]
+pub struct BenchReport(Vec<BenchResult>);
+
+impl BenchReport {
+    /// Adds a result to the report.
+    pub fn push_result(&mut self, result: BenchResult) {
+        self.0.push(result);
+    }
+
+    /// Save the report to a CSV file given by the path.
+    pub fn save(self, path: &Path) -> anyhow::Result<()> {
+        let mut writer = csv::Writer::from_path(path)?;
+        for result in self.0 {
+            writer.serialize(result)?;
+        }
+        Ok(())
+    }
+
+    /// Returns if the report has any benchmark result or not.
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+}
+
 /// Executes the command to count the gate. This command must be executed after
 /// the code is successfully compiled. To count the number of gates, we use the
 /// command `bb gates -b <target>`.

test_suite/src/bench.rs

@@ -14,3 +14,5 @@ pub const DEFAULT_DECOMPOSED_JSON_FILE: &str = "./decomposed.json";
 pub const DEFAULT_SUBSTRS_JSON_PATH: &str = "./substrs_transitions.json";
 /// Default path of the target folder for the project
 pub const DEFAULT_TARJET_JSON_FILE: &str = "./execution_project/target/execution_project.json";
+/// Default path for bench report
+pub const DEFAULT_BENCH_RESULT_FILE: &str = "./bench_result.csv";

test_suite/src/constants.rs

@@ -5,7 +5,14 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
 /// Database of regular expressions that will be tested.
-pub struct RegexDb(Vec<DbEntry>);
+#[derive(Deserialize)]
+pub struct RegexDb {
+    // Regex entries in the database
+    db_entries: Vec<DbEntry>,
+    // Defines wether we need to benchmark all the entries in the database.
+    #[serde(default)]
+    pub bench_all: bool,
+}
 
 /// Represents each fragment in a decomposed regex.
 #[derive(Deserialize, Serialize, Clone)]
@@ -119,13 +126,23 @@ impl RegexDb {
         let regex_db: Vec<DbEntry> = serde_json::from_str(&json_value["database"].to_string())
             .context("error parsing the database array")?;
 
+        // If the field in the "bench_all" is a boolean, take the boolean,
+        // otherwhise, use false as a default.
+        let bench_all: bool = match json_value["bench_all"].as_bool() {
+            Some(flag) => flag,
+            None => false,
+        };
+
         let mut regexes = Vec::new();
 
         for db_element in regex_db {
             regexes.push(db_element);
         }
 
-        Ok(Self(regexes))
+        Ok(Self {
+            db_entries: regexes,
+            bench_all,
+        })
     }
 }
 
@@ -134,7 +151,7 @@ impl IntoIterator for RegexDb {
     type IntoIter = std::vec::IntoIter<Self::Item>;
 
     fn into_iter(self) -> Self::IntoIter {
-        self.0.into_iter()
+        self.db_entries.into_iter()
     }
 }
 

test_suite/src/db.rs

@@ -5,7 +5,7 @@ mod constants;
 mod db;
 mod tester;
 
-use bench::execute_count_gate_command;
+use bench::{execute_count_gate_command, BenchReport};
 use code::Code;
 use db::RegexDb;
 use log::{self, error, info};
@@ -24,6 +24,8 @@ fn main() -> Result<(), Box<dyn Error>> {
             err
         })?;
 
+    let benchmark_all = database.bench_all;
+    let mut bench_report = BenchReport::default();
     for regex_input in database {
         info!("testing regex {}", regex_input.regex.complete_regex());
         let mut code_read_result = Code::new(&regex_input);
@@ -64,10 +66,14 @@ fn main() -> Result<(), Box<dyn Error>> {
                         None => error!("error downcasting the anyhow::Error"),
                     },
                 }
-                if regex_input.with_bench {
+                if regex_input.with_bench || benchmark_all {
                     match execute_count_gate_command() {
-                        Ok(bench_result) => {
+                        Ok(mut bench_result) => {
                             info!("benchmark results:\n{}", bench_result);
+                            // Changes the data needed to write the report.
+                            bench_result.regex = regex_input.regex.complete_regex();
+                            bench_result.with_gen_substr = regex_input.gen_substrs;
+                            bench_report.push_result(bench_result);
                         }
                         Err(err) => {
                             error!(
@@ -88,5 +94,11 @@ fn main() -> Result<(), Box<dyn Error>> {
         }
     }
 
+    // Save the bench results.
+    if !bench_report.is_empty() {
+        info!("saving benchmark results into CSV");
+        bench_report.save(Path::new(constants::DEFAULT_BENCH_RESULT_FILE))?;
+    }
+
     Ok(())
 }