DuckDB to use the Scan API

bench-orchestrator/bench_orchestrator/runner/executor.py

@@ -87,6 +87,10 @@ def run(
             else:
                 cmd = cmd_prefix + cmd
 
+        if samply and self.engine == Engine.DUCKDB:
+            # Re-use the same DuckDB instance across runs make samply output readable
+            cmd += ["--reuse"]
+
         if self.verbose:
             console.print(f"[dim]$ {' '.join(cmd)}[/dim]")
 

benchmarks/duckdb-bench/src/main.rs

@@ -73,6 +73,14 @@ struct Args {
     /// Print EXPLAIN output for each query instead of running benchmarks.
     #[arg(long, default_value_t = false)]
     explain: bool,
+
+    #[arg(
+        long,
+        default_value_t = false,
+        help = "Whether to reuse the DuckDB connection across iterations. Helpful when profiling \
+        to keep all work on the same threads"
+    )]
+    reuse: bool,
 }
 
 fn main() -> anyhow::Result<()> {
@@ -187,8 +195,11 @@ fn main() -> anyhow::Result<()> {
                 ("benchmark_name", benchmark_name.clone()),
                 ("query_idx", query_idx.to_string()),
             ]);
+
             // Make sure to reopen the duckdb connection between iterations
-            ctx.reopen()?;
+            if !args.reuse {
+                ctx.reopen()?;
+            }
             ctx.execute_query(query)
         },
     )?;

vortex-datafusion/src/v2/source.rs

@@ -278,7 +278,7 @@ impl DataSource for VortexDataSource {
         // Build the scan request with pushed-down projection, filter, and limit.
         // The projection is included so the scan can prune columns at the I/O level.
         let scan_request = ScanRequest {
-            projection: Some(self.projected_projection.clone()),
+            projection: self.projected_projection.clone(),
             filter: self.filter.clone(),
             limit: self.limit.map(|l| u64::try_from(l).unwrap_or(u64::MAX)),
             ordered: self.ordered,

vortex-duckdb/Cargo.toml

@@ -32,6 +32,7 @@ custom-labels = { workspace = true }
 futures = { workspace = true }
 glob = { workspace = true }
 itertools = { workspace = true }
+kanal = { workspace = true }
 num-traits = { workspace = true }
 object_store = { workspace = true, features = ["aws"] }
 parking_lot = { workspace = true }