Prototype of the benchmarking of the tool

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 resolver = "2"
+
 members = [
     "dctap",
     "rbe",
@@ -19,7 +20,7 @@ members = [
     "shacl_testsuite",
     "shapes_converter",
     "sparql_service",
-    "python",
+    "python"
 ]
 
 exclude = [
@@ -46,6 +47,8 @@ authors = [
 ]
 
 [workspace.dependencies]
+
+# Internal dependencies
 iri_s = { version = "0.1.32", path = "./iri_s" }
 dctap = { version = "0.1.31", path = "./dctap" }
 prefixmap = { version = "0.1.32", path = "./prefixmap" }
@@ -64,19 +67,18 @@ shex_compact = { version = "0.1.0", path = "./shex_compact" }
 srdf = { version = "0.1.35", path = "./srdf" }
 sparql_service = { version = "0.1.37", path = "./sparql_service" }
 
-# [dependencies]
 # External dependencies
-anyhow = "1.0"
-clap = { version = "4.2.1", features = ["derive"] }
-colored = "2"
-indexmap = "2.1"
-oxrdf = "0.2.0-alpha.5"
-regex = "1.10.4"
-supports-color = "3.0.0"
-serde = "1"
-serde_json = "1.0"
-serde_derive = "1"
+anyhow = "1.0.91"
+clap = { version = "4.5.20", features = ["derive"] }
+colored = "2.1.0"
+indexmap = "2.6.0"
+oxrdf = "0.2.1"
+regex = "1.11.1"
+supports-color = "3.0.1"
+serde = "1.0.214"
+serde_json = "1.0.132"
+serde_derive = "1.0.214"
 serde_yml = "0.0.12"
-thiserror = "1.0"
-tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = [ "env-filter" ] }
+thiserror = "1.0.65"
+tracing = "0.1.40"
+tracing-subscriber = { version = "0.3", features = [ "env-filter" ] }

rudof_cli/src/main.rs

@@ -33,7 +33,7 @@ use shapes_converter::ShEx2Sparql;
 use shapes_converter::{ImageFormat, ShEx2Html, ShEx2Uml, Shacl2ShEx, Tap2ShEx, UmlGenerationMode};
 use shex_ast::object_value::ObjectValue;
 use shex_ast::{ShapeExprLabel, SimpleReprSchema};
-use sparql_service::{QueryConfig, RdfData, ServiceDescription};
+use sparql_service::{LoadMode, QueryConfig, RdfData, ServiceDescription};
 use srdf::srdf_graph::SRDFGraph;
 use srdf::{
     QuerySolution, RDFFormat, RdfDataConfig, ReaderMode, SRDFBuilder, SRDFSparql, VarName, SRDF,
@@ -1062,7 +1062,7 @@ fn get_data(
         (false, None) => {
             // let data_path = cast_to_data_path(data)?;
             let data = parse_data(data, data_format, reader_mode, config)?;
-            Ok(RdfData::from_graph(data)?)
+            Ok(RdfData::from_graph(data, LoadMode::NoMaterialization)?)
         }
         (true, Some(endpoint)) => {
             let endpoint = SRDFSparql::from_str(endpoint)?;

rudof_lib/Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "rudof_lib"
 version = "0.1.45"
+documentation = "https://docs.rs/rudof_lib"
 authors.workspace = true
 description.workspace = true
-documentation = "https://docs.rs/rudof_lib"
 edition.workspace = true
 license.workspace = true
 homepage.workspace = true
@@ -13,6 +13,7 @@ repository.workspace = true
 rdf-star = ["oxrdf/rdf-star"]
 
 [dependencies]
+
 srdf = { workspace = true }
 iri_s = { workspace = true }
 shacl_ast = { workspace = true }
@@ -25,6 +26,7 @@ shapemap = { workspace = true }
 prefixmap = { workspace = true }
 shapes_converter = { workspace = true }
 dctap = { workspace = true }
+
 serde = "1"
 serde_derive = "1"
 void = "1"
@@ -35,3 +37,6 @@ itertools = "0.13"
 oxrdf = { workspace = true, features = ["oxsdatatypes"] }
 
 [dev-dependencies]
+pprof = { version = "0.13.0", features = ["flamegraph"] }
+criterion = "0.5.1"
+criterion-macro = "0.4.0"

rudof_lib/README.md

@@ -1,7 +1,23 @@
-# rudof_cli
+# rudof_lib
 
 Represents the main entry point for the rudof library.
 
 In the future, this crate could be replaced by `rudof`.
 
 The goal of this crate is to provide methods and structs that have the same functionality as we currently have in the `rudof` command line tool.
+
+## Benchmarks
+
+To enable performance profiling without running the benchmarks as root, you may need to adjust the value of `perf_event_paranoid` in the Linux kernel to an appropriate value for your environment. The most permissive value is -1.
+
+```sh
+echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid
+```
+
+For running the benchmarks using the `FlamegraphProfiler` for 5 seconds, run the following command.
+
+```sh
+cargo bench --bench <name_of_the_benchmark> -- --profile-time=5
+```
+
+We will now find a file called `flamegraph.svg` in `target/criterion/<name-of-benchmark>/profile/flamegraph.svg`.

rudof_lib/benches/perf.rs

@@ -0,0 +1,70 @@
+use std::{fs::File, os::raw::c_int, path::Path};
+
+use criterion::profiler::Profiler;
+use pprof::ProfilerGuard;
+
+/// Small custom profiler that can be used with Criterion to create a flamegraph for benchmarks.
+/// Also see [the Criterion documentation on this][custom-profiler].
+///
+/// ## Example on how to enable the custom profiler:
+///
+/// ```
+/// mod perf;
+/// use perf::FlamegraphProfiler;
+///
+/// fn fibonacci_profiled(criterion: &mut Criterion) {
+///     // Use the criterion struct as normal here.
+/// }
+///
+/// fn custom() -> Criterion {
+///     Criterion::default().with_profiler(FlamegraphProfiler::new())
+/// }
+///
+/// criterion_group! {
+///     name = benches;
+///     config = custom();
+///     targets = fibonacci_profiled
+/// }
+/// ```
+///
+/// The neat thing about this is that it will sample _only_ the benchmark, and not other stuff like
+/// the setup process.
+///
+/// Further, it will only kick in if `--profile-time <time>` is passed to the benchmark binary.
+/// A flamegraph will be created for each individual benchmark in its report directory under
+/// `profile/flamegraph.svg`.
+pub struct FlamegraphProfiler<'a> {
+    frequency: c_int,
+    active_profiler: Option<ProfilerGuard<'a>>,
+}
+
+impl<'a> FlamegraphProfiler<'a> {
+    #[allow(dead_code)]
+    pub fn new(frequency: c_int) -> Self {
+        FlamegraphProfiler {
+            frequency,
+            active_profiler: None,
+        }
+    }
+}
+
+impl<'a> Profiler for FlamegraphProfiler<'a> {
+    fn start_profiling(&mut self, _benchmark_id: &str, _benchmark_dir: &Path) {
+        self.active_profiler = Some(ProfilerGuard::new(self.frequency).unwrap());
+    }
+
+    fn stop_profiling(&mut self, _benchmark_id: &str, benchmark_dir: &Path) {
+        std::fs::create_dir_all(benchmark_dir).unwrap();
+        let flamegraph_path = benchmark_dir.join("flamegraph.svg");
+        let flamegraph_file = File::create(&flamegraph_path)
+            .expect("File system error while creating flamegraph.svg");
+        if let Some(profiler) = self.active_profiler.take() {
+            profiler
+                .report()
+                .build()
+                .unwrap()
+                .flamegraph(flamegraph_file)
+                .expect("Error writing flamegraph");
+        }
+    }
+}

rudof_lib/benches/shacl_validation.rs

@@ -0,0 +1,57 @@
+#![feature(custom_test_frameworks)]
+#![test_runner(criterion::runner)]
+
+use std::fs::File;
+use std::io::BufReader;
+
+use criterion::black_box;
+use criterion::Criterion;
+use criterion_macro::criterion;
+use rudof_lib::Rudof;
+use rudof_lib::RudofConfig;
+use rudof_lib::RudofError;
+use rudof_lib::ShaclValidationMode;
+use rudof_lib::ShapesGraphSource;
+use rudof_lib::ValidationReport;
+use shacl_ast::ShaclFormat;
+use srdf::RDFFormat;
+
+mod perf;
+
+fn custom() -> Criterion {
+    Criterion::default().with_profiler(perf::FlamegraphProfiler::new(100))
+}
+
+fn shacl_validation(rudof: &mut Rudof) -> Result<ValidationReport, RudofError> {
+    rudof.validate_shacl(
+        &ShaclValidationMode::Native,
+        &ShapesGraphSource::CurrentSchema,
+    )
+}
+
+#[criterion(custom())]
+fn bench(c: &mut Criterion) {
+    // -- SETUP --
+
+    let mut rudof = Rudof::new(&RudofConfig::default());
+
+    let reader = match File::open("../examples/book.ttl") {
+        Ok(f) => BufReader::new(f),
+        Err(_) => return,
+    };
+
+    let _ = rudof.read_shacl(reader, &ShaclFormat::Turtle, None, &srdf::ReaderMode::Lax);
+
+    let reader = match File::open("../examples/book_conformant.ttl") {
+        Ok(f) => BufReader::new(f),
+        Err(_) => return,
+    };
+
+    let _ = rudof.read_data(reader, &RDFFormat::Turtle, None, &srdf::ReaderMode::Lax);
+
+    // -- BENCH FUNCTION --
+
+    c.bench_function("SHACL Validation", |b| {
+        b.iter(|| shacl_validation(black_box(&mut rudof)))
+    });
+}

rudof_lib/src/rudof.rs

@@ -427,7 +427,7 @@ impl Rudof {
         let (compiled_schema, shacl_schema) = match shapes_graph_source {
             ShapesGraphSource::CurrentSchema if self.shacl_schema.is_some() => {
                 let ast_schema = self.shacl_schema.as_ref().unwrap();
-                let compiled_schema = ast_schema.clone().to_owned().try_into().map_err(|e| {
+                let compiled_schema = ast_schema.to_owned().try_into().map_err(|e| {
                     RudofError::SHACLCompilationError {
                         error: format!("{e}"),
                         schema: Box::new(ast_schema.clone()),

shacl_testsuite/src/manifest.rs

@@ -189,10 +189,15 @@ impl Manifest<RdfData> for GraphManifest {
     }
 
     fn load_data_graph(path: &Path, base: &str) -> RdfData {
-        Graph::from_path(Path::new(path), RDFFormat::Turtle, Some(base))
-            .unwrap()
-            .store()
-            .to_owned()
+        Graph::from_path(
+            Path::new(path),
+            RDFFormat::Turtle,
+            Some(base),
+            sparql_service::LoadMode::NoMaterialization,
+        )
+        .unwrap()
+        .store()
+        .to_owned()
     }
 
     fn base(&self) -> String {

shacl_validation/src/shacl_processor.rs

@@ -1,5 +1,6 @@
 use clap::ValueEnum;
 use shacl_ast::compiled::schema::CompiledSchema;
+use sparql_service::LoadMode;
 use sparql_service::RdfData;
 use srdf::RDFFormat;
 use srdf::SRDFBasic;
@@ -78,7 +79,7 @@ pub trait ShaclProcessor<S: SRDFBasic + Debug> {
 /// use shacl_validation::store::ShaclDataManager;
 /// use srdf::RDFFormat;
 ///
-/// let graph_validation = GraphValidation::new(
+/// let graph_validation = GraphValidation::from_path(
 ///     Path::new("../examples/book_conformant.ttl"), // example graph (refer to the examples folder)
 ///     RDFFormat::Turtle, // serialization format of the graph
 ///     None, // no base is defined
@@ -121,7 +122,7 @@ impl GraphValidation {
     /// use shacl_validation::shacl_processor::ShaclProcessor;
     /// use srdf::RDFFormat;
     ///
-    /// let graph_validation = GraphValidation::new(
+    /// let graph_validation = GraphValidation::from_path(
     ///     Path::new("../examples/book_conformant.ttl"), // example graph (refer to the examples folder)
     ///     RDFFormat::Turtle, // serialization format of the graph
     ///     None, // no base is defined
@@ -135,7 +136,15 @@ impl GraphValidation {
         mode: ShaclValidationMode,
     ) -> Result<Self, ValidateError> {
         Ok(GraphValidation {
-            store: Graph::from_path(data, data_format, base)?,
+            store: Graph::from_path(
+                data,
+                data_format,
+                base,
+                match mode {
+                    ShaclValidationMode::Native => LoadMode::NoMaterialization,
+                    ShaclValidationMode::Sparql => LoadMode::Materialization,
+                },
+            )?,
             mode,
         })
     }

shacl_validation/src/store/graph.rs

@@ -1,6 +1,6 @@
 use std::path::Path;
 
-use sparql_service::RdfData;
+use sparql_service::{LoadMode, RdfData};
 use srdf::{RDFFormat, ReaderMode, SRDFGraph};
 
 use crate::validate_error::ValidateError;
@@ -28,6 +28,7 @@ impl Graph {
         path: &Path,
         rdf_format: RDFFormat,
         base: Option<&str>,
+        load_mode: LoadMode,
     ) -> Result<Self, ValidateError> {
         match SRDFGraph::from_path(
             path,
@@ -36,15 +37,15 @@ impl Graph {
             &ReaderMode::default(), // TODO: this should be revisited
         ) {
             Ok(store) => Ok(Self {
-                store: RdfData::from_graph(store)?,
+                store: RdfData::from_graph(store, load_mode)?,
             }),
             Err(error) => Err(ValidateError::Graph(error)),
         }
     }
 
-    pub fn from_graph(graph: SRDFGraph) -> Result<Graph, ValidateError> {
+    pub fn from_graph(graph: SRDFGraph, load_mode: LoadMode) -> Result<Graph, ValidateError> {
         Ok(Graph {
-            store: RdfData::from_graph(graph)?,
+            store: RdfData::from_graph(graph, load_mode)?,
         })
     }