feat: update to DataFusion 50, pyo3 24, pyo3-arrow 0.11

alamb · alamb · commit 2ec3e1c9d8b7 · 2025-09-10T17:35:22.000-04:00
Signed-off-by: Andrew Lamb &lt;andrew@nerdnetworks.org&gt;
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -39,7 +39,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: "1.85"
+          toolchain: "1.86"
           override: true
 
       - uses: Swatinem/rust-cache@v2
@@ -91,7 +91,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: "1.85"
+          toolchain: "1.86"
           override: true
 
       - uses: Swatinem/rust-cache@v2
@@ -132,7 +132,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: "1.85"
+          toolchain: "1.86"
           override: true
 
       - uses: Swatinem/rust-cache@v2
@@ -194,7 +194,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: "1.85"
+          toolchain: "1.86"
           override: true
 
       - uses: Swatinem/rust-cache@v2
@@ -241,7 +241,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: "1.85"
+          toolchain: "1.86"
           override: true
 
       - uses: Swatinem/rust-cache@v2
@@ -300,7 +300,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: "1.85"
+          toolchain: "1.86"
           override: true
 
       - uses: Swatinem/rust-cache@v2
diff --git a/Cargo.toml b/Cargo.toml
@@ -5,7 +5,7 @@ resolver = "2"
 
 [workspace.package]
 authors = ["Qingping Hou <dave2008713@gmail.com>"]
-rust-version = "1.85"
+rust-version = "1.86"
 keywords = ["deltalake", "delta", "datalake"]
 readme = "README.md"
 edition = "2021"
@@ -26,32 +26,32 @@ debug = true
 debug = "line-tables-only"
 
 [workspace.dependencies]
-delta_kernel = { version = "0.15.1", features = [
-    "arrow-55",
+delta_kernel = { version = "0.15.2", features = [
+    "arrow-56",
     "default-engine-rustls",
     "internal-api",
 ] }
 
 
 # arrow
-arrow = { version = "55.2.0" }
-arrow-arith = { version = "55.2.0" }
-arrow-array = { version = "55.2.0", features = ["chrono-tz"] }
-arrow-buffer = { version = "55.2.0" }
-arrow-cast = { version = "55.2.0" }
-arrow-ipc = { version = "55.2.0" }
-arrow-json = { version = "55.2.0" }
-arrow-ord = { version = "55.2.0" }
-arrow-row = { version = "55.2.0" }
-arrow-schema = { version = "55.2.0" }
-arrow-select = { version = "55.2.0" }
+arrow = { version = "56.0.0" }
+arrow-arith = { version = "56.0.0" }
+arrow-array = { version = "56.0.0", features = ["chrono-tz"] }
+arrow-buffer = { version = "56.0.0" }
+arrow-cast = { version = "56.0.0" }
+arrow-ipc = { version = "56.0.0" }
+arrow-json = { version = "56.0.0" }
+arrow-ord = { version = "56.0.0" }
+arrow-row = { version = "56.0.0" }
+arrow-schema = { version = "56.0.0" }
+arrow-select = { version = "56.0.0" }
 object_store = { version = "0.12.1" }
-parquet = { version = "55.2.0" }
+parquet = { version = "56.0.0" }
 
 # datafusion
-datafusion = "49.0.0"
-datafusion-ffi = "49.0.0"
-datafusion-proto = "49.0.0"
+datafusion = "50.0.0"
+datafusion-ffi = "50.0.0"
+datafusion-proto = "50.0.0"
 
 # serde
 serde = { version = "1.0.194", features = ["derive"] }
@@ -94,3 +94,27 @@ Arro3 = "Arro3"
 AKS = "AKS"
 # to avoid using 'type' as a field name.
 tpe = "tpe"
+
+[patch.crates-io]
+datafusion = { git = "https://github.com/apache/datafusion.git", branch = "branch-50" }
+datafusion-expr = { git = "https://github.com/apache/datafusion.git", branch = "branch-50"  }
+datafusion-common = { git = "https://github.com/apache/datafusion.git", branch = "branch-50"  }
+datafusion-ffi = { git = "https://github.com/apache/datafusion.git", branch = "branch-50"  }
+datafusion-functions = { git = "https://github.com/apache/datafusion.git", branch = "branch-50"  }
+datafusion-functions-aggregate = {git = "https://github.com/apache/datafusion.git", branch = "branch-50"  }
+datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", branch = "branch-50" }
+datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", branch = "branch-50" }
+datafusion-proto = { git = "https://github.com/apache/datafusion.git", branch = "branch-50"  }
+datafusion-sql = { git = "https://github.com/apache/datafusion.git", branch = "branch-50"  }
+
+# datafusion
+# datafusion = { path = "/Users/andrewlamb/Software/datafusion2/datafusion/core" }
+# datafusion-expr = { path = "/Users/andrewlamb/Software/datafusion2/datafusion/expr"  }
+# datafusion-common = { path = "/Users/andrewlamb/Software/datafusion2/datafusion/common"  }
+# datafusion-ffi = { path = "/Users/andrewlamb/Software/datafusion2/datafusion/ffi"  }
+# datafusion-functions = { path = "/Users/andrewlamb/Software/datafusion2/datafusion/functions"  }
+# datafusion-functions-aggregate = { path = "/Users/andrewlamb/Software/datafusion2/datafusion/functions-aggregate"  }
+# datafusion-physical-expr = { path = "/Users/andrewlamb/Software/datafusion2/datafusion/physical-expr"  }
+# datafusion-physical-plan = { path = "/Users/andrewlamb/Software/datafusion2/datafusion/physical-plan"  }
+# datafusion-proto = { path = "/Users/andrewlamb/Software/datafusion2/datafusion/proto"  }
+# datafusion-sql = { path = "/Users/andrewlamb/Software/datafusion2/datafusion/sql"  }
diff --git a/crates/core/src/delta_datafusion/expr.rs b/crates/core/src/delta_datafusion/expr.rs
@@ -52,7 +52,7 @@ use crate::{DeltaResult, DeltaTableError};
 
 /// This struct is like Datafusion's MakeArray but ensures that `element` is used rather than `item
 /// as the field name within the list.
-#[derive(Debug)]
+#[derive(Debug, Hash, PartialEq, Eq)]
 struct MakeParquetArray {
     /// The actual upstream UDF, which we're just totally cheating and using
     actual: MakeArray,
diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs
@@ -389,6 +389,8 @@ pub(crate) fn get_null_of_arrow_type(t: &ArrowDataType) -> DeltaResult<ScalarVal
         )),
         //Unsupported types...
         ArrowDataType::Float16
+        | ArrowDataType::Decimal32(_, _)
+        | ArrowDataType::Decimal64(_, _)
         | ArrowDataType::Decimal256(_, _)
         | ArrowDataType::Union(_, _)
         | ArrowDataType::LargeList(_)
@@ -2185,10 +2187,10 @@ mod tests {
         assert_eq!("a", small.iter().next().unwrap().unwrap());
 
         let expected = vec![
-            ObjectStoreOperation::GetRange(LocationType::Data, 4952..4960),
-            ObjectStoreOperation::GetRange(LocationType::Data, 2399..4952),
+            ObjectStoreOperation::GetRange(LocationType::Data, 957..965),
+            ObjectStoreOperation::GetRange(LocationType::Data, 326..957),
             #[expect(clippy::single_range_in_vec_init)]
-            ObjectStoreOperation::GetRanges(LocationType::Data, vec![4..58]),
+            ObjectStoreOperation::GetRanges(LocationType::Data, vec![4..46]),
         ];
         let mut actual = Vec::new();
         operations.recv_many(&mut actual, 3).await;
diff --git a/crates/core/src/kernel/scalars.rs b/crates/core/src/kernel/scalars.rs
@@ -250,6 +250,8 @@ impl ScalarExt for Scalar {
                 ))
             }
             Float16
+            | Decimal32(_, _)
+            | Decimal64(_, _)
             | Decimal256(_, _)
             | List(_)
             | LargeList(_)
diff --git a/crates/core/src/operations/optimize.rs b/crates/core/src/operations/optimize.rs
@@ -1111,7 +1111,7 @@ pub(super) mod zorder {
         }
 
         // DataFusion UDF impl for zorder_key
-        #[derive(Debug)]
+        #[derive(Debug, Hash, PartialEq, Eq)]
         pub struct ZOrderUDF;
 
         impl ScalarUDFImpl for ZOrderUDF {
diff --git a/python/Cargo.toml b/python/Cargo.toml
@@ -20,7 +20,7 @@ doc = false
 [dependencies]
 delta_kernel.workspace = true
 
-pyo3-arrow = { version = "0.9.0", default-features = false }
+pyo3-arrow = { version = "0.11.0", default-features = false }
 
 # arrow
 arrow-schema = { workspace = true, features = ["serde"] }
@@ -63,7 +63,7 @@ jemallocator = { version = "0.5", features = [
 jemallocator = { version = "0.5", features = ["disable_initial_exec_tls"] }
 
 [dependencies.pyo3]
-version = "0.24.0"
+version = "0.25.1"
 features = ["extension-module", "abi3", "abi3-py39"]
 
 [dependencies.deltalake]
diff --git a/python/src/datafusion.rs b/python/src/datafusion.rs
@@ -175,6 +175,10 @@ mod tests {
     }
 
     impl LazyBatchGenerator for TestBatchGenerator {
+        fn as_any(&self) -> &dyn Any {
+            self
+        }
+
         fn generate_next_batch(&mut self) -> DataFusionResult<Option<RecordBatch>> {
             if self.current_index < self.data.len() {
                 let batch = self.data[self.current_index].clone();
diff --git a/python/src/writer.rs b/python/src/writer.rs
@@ -1,4 +1,6 @@
 //! This module contains helper functions to create a LazyTableProvider from an ArrowArrayStreamReader
+
+use std::any::Any;
 use std::fmt::{self};
 use std::sync::{Arc, Mutex};
 
@@ -66,6 +68,10 @@ impl ArrowStreamBatchGenerator {
 }
 
 impl LazyBatchGenerator for ArrowStreamBatchGenerator {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
     fn generate_next_batch(
         &mut self,
     ) -> deltalake::datafusion::error::Result<Option<deltalake::arrow::array::RecordBatch>> {

Original file line number	Diff line number	Diff line change
`@@ -250,6 +250,8 @@ impl ScalarExt for Scalar {`
`250`	`250`	`))`
`251`	`251`	`}`
`252`	`252`	`Float16`
	`253`	`+ \| Decimal32(_, _)`
	`254`	`+ \| Decimal64(_, _)`
`253`	`255`	`\| Decimal256(_, _)`
`254`	`256`	`\| List(_)`
`255`	`257`	`\| LargeList(_)`
Original file line number	Diff line number	Diff line change
`@@ -1111,7 +1111,7 @@ pub(super) mod zorder {`
`1111`	`1111`	`}`
`1112`	`1112`
`1113`	`1113`	`// DataFusion UDF impl for zorder_key`
`1114`		`- #[derive(Debug)]`
	`1114`	`+ #[derive(Debug, Hash, PartialEq, Eq)]`
`1115`	`1115`	`pub struct ZOrderUDF;`
`1116`	`1116`
`1117`	`1117`	`impl ScalarUDFImpl for ZOrderUDF {`