fix: sample multiples of 1024

vortex-btrblocks/src/integer.rs

@@ -828,7 +828,7 @@ mod tests {
 
         let numbers = [0, 10, 50, 100, 1000, 3000]
             .into_iter()
-            .map(|i| 1234 * i)
+            .map(|i| 12340 * i) // must be big enough to not prefer fastlanes.bitpacked
             .collect_vec();
 
         let mut rng = StdRng::seed_from_u64(1u64);
@@ -999,7 +999,7 @@ mod scheme_selection_tests {
         let mut codes = Vec::with_capacity(65_535);
         let numbers: Vec<i32> = [0, 10, 50, 100, 1000, 3000]
             .into_iter()
-            .map(|i| 1234 * i)
+            .map(|i| 12340 * i) // must be big enough to not prefer fastlanes.bitpacked
             .collect();
 
         let mut rng = StdRng::seed_from_u64(1u64);
@@ -1020,7 +1020,7 @@ mod scheme_selection_tests {
     fn test_runend_compressed() {
         let mut values: Vec<i32> = Vec::new();
         for i in 0..100 {
-            values.extend(iter::repeat_n(1_000_000 + i, 10));
+            values.extend(iter::repeat_n((i32::MAX - 50).wrapping_add(i), 10));
         }
         let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable);
         let compressed = IntCompressor::compress(&array, false, 3, &[]).unwrap();

vortex-btrblocks/src/lib.rs

@@ -93,7 +93,15 @@ impl Default for GenerateStatsOptions {
     }
 }
 
+/// The size of each sampled run.
 const SAMPLE_SIZE: u32 = 64;
+/// The number of sampled runs.
+///
+/// # Warning
+///
+/// The product of SAMPLE_SIZE and SAMPLE_COUNT should be (roughly) a multiple of 1024 so that
+/// fastlanes bitpacking of sampled vectors does not introduce (large amounts of) padding.
+const SAMPLE_COUNT: u32 = 16;
 
 /// Stats for the compressor.
 pub trait CompressorStats: Debug + Clone {
@@ -186,25 +194,26 @@ fn estimate_compression_ratio_with_sampling<T: Scheme + ?Sized>(
         // We want to sample about 1% of data
         let source_len = stats.source().len();
 
-        // We want to sample about 1% of data, while keeping a minimal sample of 640 values.
-        let sample_count = usize::max(
-            (source_len / 100)
-                / usize::try_from(SAMPLE_SIZE).vortex_expect("SAMPLE_SIZE must fit in usize"),
-            10,
+        // We want to sample about 1% of data, while keeping a minimal sample of 1024 values.
+        let approximately_one_percent = (source_len / 100)
+            / usize::try_from(SAMPLE_SIZE).vortex_expect("SAMPLE_SIZE must fit in usize");
+        let sample_count = u32::max(
+            u32::next_multiple_of(
+                approximately_one_percent
+                    .try_into()
+                    .vortex_expect("sample count must fit in u32"),
+                16,
+            ),
+            SAMPLE_COUNT,
         );
 
         tracing::trace!(
             "Sampling {} values out of {}",
-            SAMPLE_SIZE as usize * sample_count,
+            SAMPLE_SIZE as u64 * sample_count as u64,
             source_len
         );
 
-        stats.sample(
-            SAMPLE_SIZE,
-            sample_count
-                .try_into()
-                .vortex_expect("sample count must fit in u32"),
-        )
+        stats.sample(SAMPLE_SIZE, sample_count)
     };
 
     let after = compressor