Merge branch 'main' into alamb/refactor_push_decoder

Author: alamb

.github/actions/setup-builder/action.yaml

@@ -16,7 +16,7 @@
 # under the License.
 
 name: Prepare Rust Builder
-description: 'Prepare Rust Build Environment'
+description: "Prepare Rust Build Environment"
 runs:
   using: "composite"
   steps:
@@ -53,6 +53,9 @@ runs:
     - name: Enable backtraces
       shell: bash
       run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
+    - name: Disable incremental compilation
+      shell: bash
+      run: echo CARGO_INCREMENTAL=0 >> $GITHUB_ENV
     - name: Fixup git permissions
       # https://github.com/actions/checkout/issues/766
       shell: bash

arrow-array/Cargo.toml

@@ -25,7 +25,7 @@ authors = { workspace = true }
 license = { workspace = true }
 keywords = { workspace = true }
 include = { workspace = true }
-edition = { workspace = true }
+edition = "2024"
 rust-version = { workspace = true }
 
 [lib]

arrow-array/benches/fixed_size_list_array.rs

@@ -18,7 +18,7 @@
 use arrow_array::{Array, FixedSizeListArray, Int32Array};
 use arrow_schema::Field;
 use criterion::*;
-use rand::{rng, Rng};
+use rand::{Rng, rng};
 use std::{hint, sync::Arc};
 
 fn gen_fsl(len: usize, value_len: usize) -> FixedSizeListArray {

arrow-array/benches/occupancy.rs

@@ -19,7 +19,7 @@ use arrow_array::types::Int32Type;
 use arrow_array::{DictionaryArray, Int32Array};
 use arrow_buffer::NullBuffer;
 use criterion::*;
-use rand::{rng, Rng};
+use rand::{Rng, rng};
 use std::{hint, sync::Arc};
 
 fn gen_dict(

arrow-array/benches/union_array.rs

@@ -21,7 +21,7 @@ use arrow_array::{Array, ArrayRef, Int32Array, UnionArray};
 use arrow_buffer::{NullBuffer, ScalarBuffer};
 use arrow_schema::{DataType, Field, UnionFields};
 use criterion::*;
-use rand::{rng, Rng};
+use rand::{Rng, rng};
 
 fn array_with_nulls() -> ArrayRef {
     let mut rng = rng();

arrow-array/src/arithmetic.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use arrow_buffer::{i256, ArrowNativeType, IntervalDayTime, IntervalMonthDayNano};
+use arrow_buffer::{ArrowNativeType, IntervalDayTime, IntervalMonthDayNano, i256};
 use arrow_schema::ArrowError;
 use half::f16;
 use num_complex::ComplexFloat;
@@ -454,9 +454,7 @@ mod tests {
     use super::*;
 
     macro_rules! assert_approx_eq {
-        ( $x: expr, $y: expr ) => {{
-            assert_approx_eq!($x, $y, 1.0e-4)
-        }};
+        ( $x: expr, $y: expr ) => {{ assert_approx_eq!($x, $y, 1.0e-4) }};
         ( $x: expr, $y: expr, $tol: expr ) => {{
             let x_val = $x;
             let y_val = $y;

arrow-array/src/array/binary_array.rs

@@ -90,7 +90,7 @@ impl<OffsetSize: OffsetSizeTrait> GenericBinaryArray<OffsetSize> {
         &'a self,
         indexes: impl Iterator<Item = Option<usize>> + 'a,
     ) -> impl Iterator<Item = Option<&'a [u8]>> {
-        indexes.map(|opt_index| opt_index.map(|index| self.value_unchecked(index)))
+        unsafe { indexes.map(|opt_index| opt_index.map(|index| self.value_unchecked(index))) }
     }
 }
 

arrow-array/src/array/boolean_array.rs

@@ -19,7 +19,7 @@ use crate::array::print_long_array;
 use crate::builder::BooleanBuilder;
 use crate::iterator::BooleanIter;
 use crate::{Array, ArrayAccessor, ArrayRef, Scalar};
-use arrow_buffer::{bit_util, BooleanBuffer, Buffer, MutableBuffer, NullBuffer};
+use arrow_buffer::{BooleanBuffer, Buffer, MutableBuffer, NullBuffer, bit_util};
 use arrow_data::{ArrayData, ArrayDataBuilder};
 use arrow_schema::DataType;
 use std::any::Any;
@@ -184,7 +184,7 @@ impl BooleanArray {
     /// # Safety
     /// This doesn't check bounds, the caller must ensure that index < self.len()
     pub unsafe fn value_unchecked(&self, i: usize) -> bool {
-        self.values.value_unchecked(i)
+        unsafe { self.values.value_unchecked(i) }
     }
 
     /// Returns the boolean value at index `i`.
@@ -222,7 +222,7 @@ impl BooleanArray {
         &'a self,
         indexes: impl Iterator<Item = Option<usize>> + 'a,
     ) -> impl Iterator<Item = Option<bool>> + 'a {
-        indexes.map(|opt_index| opt_index.map(|index| self.value_unchecked(index)))
+        indexes.map(|opt_index| opt_index.map(|index| unsafe { self.value_unchecked(index) }))
     }
 
     /// Create a [`BooleanArray`] by evaluating the operation for
@@ -355,7 +355,7 @@ impl ArrayAccessor for &BooleanArray {
     }
 
     unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
-        BooleanArray::value_unchecked(self, index)
+        unsafe { BooleanArray::value_unchecked(self, index) }
     }
 }
 
@@ -486,7 +486,7 @@ impl From<BooleanBuffer> for BooleanArray {
 mod tests {
     use super::*;
     use arrow_buffer::Buffer;
-    use rand::{rng, Rng};
+    use rand::{Rng, rng};
 
     #[test]
     fn test_boolean_fmt_debug() {

arrow-array/src/array/byte_array.rs

@@ -18,8 +18,8 @@
 use crate::array::{get_offsets, print_long_array};
 use crate::builder::GenericByteBuilder;
 use crate::iterator::ArrayIter;
-use crate::types::bytes::ByteArrayNativeType;
 use crate::types::ByteArrayType;
+use crate::types::bytes::ByteArrayNativeType;
 use crate::{Array, ArrayAccessor, ArrayRef, OffsetSizeTrait, Scalar};
 use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer};
 use arrow_buffer::{NullBuffer, OffsetBuffer};
@@ -283,8 +283,8 @@ impl<T: ByteArrayType> GenericByteArray<T> {
     /// # Safety
     /// Caller is responsible for ensuring that the index is within the bounds of the array
     pub unsafe fn value_unchecked(&self, i: usize) -> &T::Native {
-        let end = *self.value_offsets().get_unchecked(i + 1);
-        let start = *self.value_offsets().get_unchecked(i);
+        let end = *unsafe { self.value_offsets().get_unchecked(i + 1) };
+        let start = *unsafe { self.value_offsets().get_unchecked(i) };
 
         // Soundness
         // pointer alignment & location is ensured by RawPtrBox
@@ -295,16 +295,18 @@ impl<T: ByteArrayType> GenericByteArray<T> {
         // OffsetSizeTrait. Currently, only i32 and i64 implement OffsetSizeTrait,
         // both of which should cleanly cast to isize on an architecture that supports
         // 32/64-bit offsets
-        let b = std::slice::from_raw_parts(
-            self.value_data
-                .as_ptr()
-                .offset(start.to_isize().unwrap_unchecked()),
-            (end - start).to_usize().unwrap_unchecked(),
-        );
+        let b = unsafe {
+            std::slice::from_raw_parts(
+                self.value_data
+                    .as_ptr()
+                    .offset(start.to_isize().unwrap_unchecked()),
+                (end - start).to_usize().unwrap_unchecked(),
+            )
+        };
 
         // SAFETY:
         // ArrayData is valid
-        T::Native::from_bytes_unchecked(b)
+        unsafe { T::Native::from_bytes_unchecked(b) }
     }
 
     /// Returns the element at index `i`
@@ -509,7 +511,7 @@ impl<'a, T: ByteArrayType> ArrayAccessor for &'a GenericByteArray<T> {
     }
 
     unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
-        GenericByteArray::value_unchecked(self, index)
+        unsafe { GenericByteArray::value_unchecked(self, index) }
     }
 }
 
@@ -603,14 +605,23 @@ mod tests {
         let nulls = NullBuffer::new_null(3);
         let err =
             StringArray::try_new(offsets.clone(), data.clone(), Some(nulls.clone())).unwrap_err();
-        assert_eq!(err.to_string(), "Invalid argument error: Incorrect length of null buffer for StringArray, expected 2 got 3");
+        assert_eq!(
+            err.to_string(),
+            "Invalid argument error: Incorrect length of null buffer for StringArray, expected 2 got 3"
+        );
 
         let err = BinaryArray::try_new(offsets.clone(), data.clone(), Some(nulls)).unwrap_err();
-        assert_eq!(err.to_string(), "Invalid argument error: Incorrect length of null buffer for BinaryArray, expected 2 got 3");
+        assert_eq!(
+            err.to_string(),
+            "Invalid argument error: Incorrect length of null buffer for BinaryArray, expected 2 got 3"
+        );
 
         let non_utf8_data = Buffer::from_slice_ref(b"he\xFFloworld");
         let err = StringArray::try_new(offsets.clone(), non_utf8_data.clone(), None).unwrap_err();
-        assert_eq!(err.to_string(), "Invalid argument error: Encountered non UTF-8 data: invalid utf-8 sequence of 1 bytes from index 2");
+        assert_eq!(
+            err.to_string(),
+            "Invalid argument error: Encountered non UTF-8 data: invalid utf-8 sequence of 1 bytes from index 2"
+        );
 
         BinaryArray::new(offsets, non_utf8_data, None);
 

arrow-array/src/array/byte_view_array.rs

@@ -324,17 +324,17 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
     /// Caller is responsible for ensuring that the index is within the bounds
     /// of the array
     pub unsafe fn value_unchecked(&self, idx: usize) -> &T::Native {
-        let v = self.views.get_unchecked(idx);
+        let v = unsafe { self.views.get_unchecked(idx) };
         let len = *v as u32;
         let b = if len <= MAX_INLINE_VIEW_LEN {
-            Self::inline_value(v, len as usize)
+            unsafe { Self::inline_value(v, len as usize) }
         } else {
             let view = ByteView::from(*v);
-            let data = self.buffers.get_unchecked(view.buffer_index as usize);
+            let data = unsafe { self.buffers.get_unchecked(view.buffer_index as usize) };
             let offset = view.offset as usize;
-            data.get_unchecked(offset..offset + len as usize)
+            unsafe { data.get_unchecked(offset..offset + len as usize) }
         };
-        T::Native::from_bytes_unchecked(b)
+        unsafe { T::Native::from_bytes_unchecked(b) }
     }
 
     /// Returns the first `len` bytes the inline value of the view.
@@ -345,7 +345,9 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
     #[inline(always)]
     pub unsafe fn inline_value(view: &u128, len: usize) -> &[u8] {
         debug_assert!(len <= MAX_INLINE_VIEW_LEN as usize);
-        std::slice::from_raw_parts((view as *const u128 as *const u8).wrapping_add(4), len)
+        unsafe {
+            std::slice::from_raw_parts((view as *const u128 as *const u8).wrapping_add(4), len)
+        }
     }
 
     /// Constructs a new iterator for iterating over the values of this array
@@ -541,7 +543,7 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
     #[inline(always)]
     unsafe fn copy_view_to_buffer(&self, i: usize, data_buf: &mut Vec<u8>) -> u128 {
         // SAFETY: `i < self.len()` ensures this is in‑bounds.
-        let raw_view = *self.views().get_unchecked(i);
+        let raw_view = unsafe { *self.views().get_unchecked(i) };
         let mut bv = ByteView::from(raw_view);
 
         // Inline‑small views stay as‑is.
@@ -550,10 +552,10 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
         } else {
             // SAFETY: `bv.buffer_index` and `bv.offset..bv.offset+bv.length`
             // must both lie within valid ranges for `self.buffers`.
-            let buffer = self.buffers.get_unchecked(bv.buffer_index as usize);
+            let buffer = unsafe { self.buffers.get_unchecked(bv.buffer_index as usize) };
             let start = bv.offset as usize;
             let end = start + bv.length as usize;
-            let slice = buffer.get_unchecked(start..end);
+            let slice = unsafe { buffer.get_unchecked(start..end) };
 
             // Copy out‑of‑line data into our single “0” buffer.
             let new_offset = data_buf.len() as u32;
@@ -624,10 +626,10 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
         right: &GenericByteViewArray<T>,
         right_idx: usize,
     ) -> Ordering {
-        let l_view = left.views().get_unchecked(left_idx);
+        let l_view = unsafe { left.views().get_unchecked(left_idx) };
         let l_byte_view = ByteView::from(*l_view);
 
-        let r_view = right.views().get_unchecked(right_idx);
+        let r_view = unsafe { right.views().get_unchecked(right_idx) };
         let r_byte_view = ByteView::from(*r_view);
 
         let l_len = l_byte_view.length;
@@ -853,7 +855,7 @@ impl<'a, T: ByteViewType + ?Sized> ArrayAccessor for &'a GenericByteViewArray<T>
     }
 
     unsafe fn value_unchecked(&self, index: usize) -> Self::Item {
-        GenericByteViewArray::value_unchecked(self, index)
+        unsafe { GenericByteViewArray::value_unchecked(self, index) }
     }
 }
 
@@ -999,7 +1001,7 @@ impl BinaryViewArray {
     /// # Safety
     /// Caller is responsible for ensuring that items in array are utf8 data.
     pub unsafe fn to_string_view_unchecked(self) -> StringViewArray {
-        StringViewArray::new_unchecked(self.views, self.buffers, self.nulls)
+        unsafe { StringViewArray::new_unchecked(self.views, self.buffers, self.nulls) }
     }
 }
 
@@ -1171,7 +1173,10 @@ mod tests {
             builder.finish()
         };
         assert_eq!(array.value(0), "large payload over 12 bytes");
-        assert_eq!(array.value(1), "another large payload over 12 bytes that double than the first one, so that we can trigger the in_progress in builder re-created");
+        assert_eq!(
+            array.value(1),
+            "another large payload over 12 bytes that double than the first one, so that we can trigger the in_progress in builder re-created"
+        );
         assert_eq!(2, array.buffers.len());
     }