Operator Updates

Cargo.toml

@@ -119,7 +119,6 @@ jiff = "0.2.0"
 kanal = "0.1.1"
 lending-iterator = "0.1.7"
 libfuzzer-sys = "0.4"
-linked_hash_set = "0.1.5"
 log = { version = "0.4.21" }
 loom = { version = "0.7", features = ["checkpoint"] }
 memmap2 = "0.9.5"

encodings/fastlanes/benches/pipeline_bitpacking.rs

@@ -10,7 +10,6 @@ use mimalloc::MiMalloc;
 use rand::prelude::StdRng;
 use rand::{Rng, SeedableRng};
 use vortex_array::compute::{filter, warm_up_vtables};
-use vortex_array::pipeline::{Element, export_canonical_pipeline_expr};
 use vortex_array::{IntoArray, ToCanonical};
 use vortex_buffer::BufferMut;
 use vortex_dtype::NativePType;
@@ -68,32 +67,33 @@ pub fn decompress_bitpacking_late_filter<T: NativePType>(bencher: Bencher, fract
         .bench_values(|mask| filter(array.to_canonical().as_ref(), &mask).unwrap());
 }
 
-#[divan::bench(types = [i8, i16, i32, i64], args = TRUE_COUNT)]
-pub fn decompress_bitpacking_pipeline_filter<T: Element + NativePType>(
-    bencher: Bencher,
-    fraction_kept: f64,
-) {
-    let mut rng = StdRng::seed_from_u64(0);
-    let values = (0..LENGTH)
-        .map(|_| T::from(rng.random_range(0..100)).unwrap())
-        .collect::<BufferMut<T>>()
-        .into_array()
-        .to_primitive();
-    let array = bitpack_to_best_bit_width(&values).unwrap();
-
-    let mask = (0..LENGTH)
-        .map(|_| rng.random_bool(fraction_kept))
-        .collect::<BooleanBuffer>();
-
-    bencher
-        .with_inputs(|| Mask::from_buffer(mask.clone()))
-        .bench_local_values(|mask| {
-            export_canonical_pipeline_expr(
-                array.dtype(),
-                array.len(),
-                array.to_operator().unwrap().unwrap().as_ref(),
-                &mask,
-            )
-            .unwrap()
-        });
-}
+// TODO(ngates): bring back benchmarks once operator API is stable.
+// #[divan::bench(types = [i8, i16, i32, i64], args = TRUE_COUNT)]
+// pub fn decompress_bitpacking_pipeline_filter<T: Element + NativePType>(
+//     bencher: Bencher,
+//     fraction_kept: f64,
+// ) {
+//     let mut rng = StdRng::seed_from_u64(0);
+//     let values = (0..LENGTH)
+//         .map(|_| T::from(rng.random_range(0..100)).unwrap())
+//         .collect::<BufferMut<T>>()
+//         .into_array()
+//         .to_primitive();
+//     let array = bitpack_to_best_bit_width(&values).unwrap();
+//
+//     let mask = (0..LENGTH)
+//         .map(|_| rng.random_bool(fraction_kept))
+//         .collect::<BooleanBuffer>();
+//
+//     bencher
+//         .with_inputs(|| Mask::from_buffer(mask.clone()))
+//         .bench_local_values(|mask| {
+//             export_canonical_pipeline_expr(
+//                 array.dtype(),
+//                 array.len(),
+//                 array.to_operator().unwrap().unwrap().as_ref(),
+//                 &mask,
+//             )
+//             .unwrap()
+//         });
+// }

encodings/fastlanes/benches/pipeline_bitpacking_compare_scalar.rs

@@ -10,13 +10,11 @@ use mimalloc::MiMalloc;
 use rand::prelude::StdRng;
 use rand::{Rng, SeedableRng};
 use vortex_array::compute::{filter, warm_up_vtables};
-use vortex_array::pipeline::{Element, export_canonical_pipeline_expr};
 use vortex_array::{Array, ArrayRef, IntoArray, ToCanonical};
 use vortex_buffer::BufferMut;
-use vortex_dtype::Nullability::NonNullable;
-use vortex_dtype::{DType, NativePType};
+use vortex_dtype::NativePType;
 use vortex_error::VortexResult;
-use vortex_expr::{Scope, VortexExprExt, lit, lt, root};
+use vortex_expr::{Scope, lit, lt, root};
 use vortex_fastlanes::{FoRArray, bitpack_to_best_bit_width};
 use vortex_mask::Mask;
 use vortex_scalar::Scalar;
@@ -74,58 +72,59 @@ pub fn eval<T: NativePType + Into<Scalar>>(bencher: Bencher, fraction_kept: f64)
         });
 }
 
-#[divan::bench(types = [u8, u16, u32, u64], args = TRUE_COUNT)]
-pub fn pipeline<T: Element + NativePType + Into<Scalar>>(bencher: Bencher, fraction_kept: f64) {
-    let mut rng = StdRng::seed_from_u64(0);
-    let values = (0..100_000)
-        .map(|_| T::from(rng.random_range(10..100)).unwrap())
-        .collect::<BufferMut<T>>();
-    let array = create_for_bitpacked_array(values).unwrap();
-
-    let mask = (0..100_000)
-        .map(|_| rng.random_bool(fraction_kept))
-        .collect::<BooleanBuffer>();
-
-    let expr = lt(root(), lit(T::from_i32(2).unwrap()));
-    let operator = expr.to_operator_unoptimized(&array).unwrap().unwrap();
-
-    bencher
-        .with_inputs(|| Mask::from_buffer(mask.clone()))
-        .bench_local_values(|mask| {
-            export_canonical_pipeline_expr(
-                &DType::Bool(NonNullable),
-                array.len(),
-                operator.as_ref(),
-                &mask,
-            )
-            .unwrap()
-        });
-}
-
-#[divan::bench(types = [u8, u16, u32, u64], args = TRUE_COUNT)]
-pub fn pipeline_opt<T: Element + NativePType + Into<Scalar>>(bencher: Bencher, fraction_kept: f64) {
-    let mut rng = StdRng::seed_from_u64(0);
-    let values = (0..100_000)
-        .map(|_| T::from(rng.random_range(10..100)).unwrap())
-        .collect::<BufferMut<T>>();
-    let array = create_for_bitpacked_array(values).unwrap();
-
-    let mask = (0..100_000)
-        .map(|_| rng.random_bool(fraction_kept))
-        .collect::<BooleanBuffer>();
-
-    let expr = lt(root(), lit(T::from_i32(2).unwrap()));
-    let operator = expr.to_operator(&array).unwrap().unwrap();
-
-    bencher
-        .with_inputs(|| (Mask::from_buffer(mask.clone()), operator.clone()))
-        .bench_local_values(|(mask, operator)| {
-            export_canonical_pipeline_expr(
-                &DType::Bool(NonNullable),
-                array.len(),
-                operator.as_ref(),
-                &mask,
-            )
-            .unwrap()
-        });
-}
+// TODO(ngates): bring back benchmarks once operator API is stable.
+// #[divan::bench(types = [u8, u16, u32, u64], args = TRUE_COUNT)]
+// pub fn operator<T: Element + NativePType + Into<Scalar>>(bencher: Bencher, fraction_kept: f64) {
+//     let mut rng = StdRng::seed_from_u64(0);
+//     let values = (0..100_000)
+//         .map(|_| T::from(rng.random_range(10..100)).unwrap())
+//         .collect::<BufferMut<T>>();
+//     let array = create_for_bitpacked_array(values).unwrap();
+//
+//     let mask = (0..100_000)
+//         .map(|_| rng.random_bool(fraction_kept))
+//         .collect::<BooleanBuffer>();
+//
+//     let expr = lt(root(), lit(T::from_i32(2).unwrap()));
+//     let operator = expr.to_operator_unoptimized(&array).unwrap().unwrap();
+//
+//     bencher
+//         .with_inputs(|| Mask::from_buffer(mask.clone()))
+//         .bench_local_values(|mask| {
+//             export_canonical_pipeline_expr(
+//                 &DType::Bool(NonNullable),
+//                 array.len(),
+//                 operator.as_ref(),
+//                 &mask,
+//             )
+//             .unwrap()
+//         });
+// }
+//
+// #[divan::bench(types = [u8, u16, u32, u64], args = TRUE_COUNT)]
+// pub fn pipeline_opt<T: Element + NativePType + Into<Scalar>>(bencher: Bencher, fraction_kept: f64) {
+//     let mut rng = StdRng::seed_from_u64(0);
+//     let values = (0..100_000)
+//         .map(|_| T::from(rng.random_range(10..100)).unwrap())
+//         .collect::<BufferMut<T>>();
+//     let array = create_for_bitpacked_array(values).unwrap();
+//
+//     let mask = (0..100_000)
+//         .map(|_| rng.random_bool(fraction_kept))
+//         .collect::<BooleanBuffer>();
+//
+//     let expr = lt(root(), lit(T::from_i32(2).unwrap()));
+//     let operator = expr.to_operator(&array).unwrap().unwrap();
+//
+//     bencher
+//         .with_inputs(|| (Mask::from_buffer(mask.clone()), operator.clone()))
+//         .bench_local_values(|(mask, operator)| {
+//             export_canonical_pipeline_expr(
+//                 &DType::Bool(NonNullable),
+//                 array.len(),
+//                 operator.as_ref(),
+//                 &mask,
+//             )
+//             .unwrap()
+//         });
+// }