[WIP] Bump Apache DF to 55

Cargo.toml

@@ -92,7 +92,7 @@ keywords = ["vortex"]
 license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/spiraldb/vortex"
-rust-version = "1.91.0"
+rust-version = "1.95.0"
 version = "0.1.0"
 
 [workspace.dependencies]
@@ -101,16 +101,16 @@ anyhow = "1.0.100"
 arbitrary = "1.3.2"
 arc-swap = "1.9"
 arcref = "0.2.0"
-arrow-arith = "58.3"
-arrow-array = "58.3"
-arrow-buffer = "58.3"
-arrow-cast = "58.3"
-arrow-data = "58.3"
-arrow-ipc = "58.3"
-arrow-row = "58.3"
-arrow-schema = "58.3"
-arrow-select = "58.3"
-arrow-string = "58.3"
+arrow-arith = "59.2"
+arrow-array = "59.2"
+arrow-buffer = "59.2"
+arrow-cast = "59.2"
+arrow-data = "59.2"
+arrow-ipc = "59.2"
+arrow-row = "59.2"
+arrow-schema = "59.2"
+arrow-select = "59.2"
+arrow-string = "59.2"
 async-fs = "2.2.0"
 async-lock = "3.4"
 async-stream = "0.3.6"
@@ -203,9 +203,9 @@ opentelemetry = "0.32.0"
 opentelemetry-otlp = "0.32.0"
 opentelemetry_sdk = "0.32.0"
 parking_lot = { version = "0.12.3", features = ["nightly"] }
-parquet = "58.3"
-parquet-variant = "58.3"
-parquet-variant-compute = "58.3"
+parquet = "59.2"
+parquet-variant = "59.2"
+parquet-variant-compute = "59.2"
 paste = "1.0.15"
 pco = "1.0.1"
 percent-encoding = "2.3.2"
@@ -439,3 +439,43 @@ incremental = false
 # This improved build times significantly for default common cases that we use locally
 [profile.dev.package.vortex-fastlanes]
 debug = false
+
+[patch.crates-io]
+datafusion = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-catalog = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-catalog-listing = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-common-runtime = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-datasource = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-datasource-arrow = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-datasource-avro = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-datasource-csv = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-datasource-json = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-datasource-parquet = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-doc = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-execution = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-expr = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-expr-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-functions = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-functions-nested = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-functions-table = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-functions-window = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-functions-window-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-macros = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-optimizer = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-physical-expr = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-physical-optimizer = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-physical-plan = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-pruning = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-session = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-spark = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-sql = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+datafusion-sqllogictest = { git = "https://github.com/apache/datafusion", rev = "2bfdd4aea181525f5328781b29174d82be0c7fd4" }
+geoarrow = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }
+geoarrow-array = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }
+geoarrow-cast = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }
+geoarrow-schema = { version = "=0.8.0", git = "https://github.com/geoarrow/geoarrow-rs", rev = "855d2e138e54ce503c4488d5bb795ca1ea880d1f" }

rust-toolchain.toml

@@ -1,4 +1,4 @@
 [toolchain]
-channel = "1.91.0"
+channel = "1.95.0"
 components = ["rust-src", "rustfmt", "clippy", "rust-analyzer"]
 profile = "minimal"

vortex-array/src/arrays/chunked/compute/filter.rs

@@ -71,7 +71,7 @@ fn filter_slices(
     let chunk_filters = chunk_filters(array, slices)?;
 
     // Now, apply the chunk filter to every slice.
-    for (chunk, chunk_filter) in array.iter_chunks().zip(chunk_filters.into_iter()) {
+    for (chunk, chunk_filter) in array.iter_chunks().zip(chunk_filters) {
         match chunk_filter {
             // All => preserve the entire chunk unfiltered.
             ChunkFilter::All => result.push(chunk.clone()),

vortex-array/src/arrays/decimal/array.rs

@@ -609,7 +609,7 @@ where
         )
     }
 
-    for (idx, value) in patch_indices.iter().zip_eq(patch_values.into_iter()) {
+    for (idx, value) in patch_indices.iter().zip_eq(patch_values) {
         buffer[idx.as_() - patch_indices_offset] = <ValuesDVT as BigCast>::from(value).vortex_expect(
             "values of a given DecimalDType are representable in all compatible NativeDecimalType",
         );

vortex-arrow/src/dtype.rs

@@ -372,11 +372,10 @@ pub(crate) fn to_data_type_naive(dtype: &DType) -> VortexResult<DataType> {
             let scale = dt.scale();
 
             match precision {
-                // This code is commented out until DataFusion improves its support for smaller decimals.
                 // // DECIMAL32_MAX_PRECISION
-                // 0..=9 => DataType::Decimal32(precision, scale),
+                0..=9 => DataType::Decimal32(precision, scale),
                 // // DECIMAL64_MAX_PRECISION
-                // 10..=18 => DataType::Decimal64(precision, scale),
+                10..=18 => DataType::Decimal64(precision, scale),
                 // DECIMAL128_MAX_PRECISION
                 0..=38 => DataType::Decimal128(precision, scale),
                 // DECIMAL256_MAX_PRECISION

vortex-cuda/build.rs

@@ -74,12 +74,10 @@ fn main() {
                 .is_some_and(|n| n.starts_with("bit_unpack_"));
 
             match path.extension().and_then(|e| e.to_str()) {
-                Some("cuh") | Some("h") => {
-                    // Only watch hand-written .cuh/.h files, not generated ones
-                    // (generated files are rebuilt when cuda_kernel_generator changes)
-                    if !is_generated {
-                        println!("cargo:rerun-if-changed={}", path.display());
-                    }
+                // Only watch hand-written .cuh/.h files, not generated ones
+                // (generated files are rebuilt when cuda_kernel_generator changes)
+                Some("cuh") | Some("h") if !is_generated => {
+                    println!("cargo:rerun-if-changed={}", path.display());
                 }
                 Some("cu") => {
                     // Only watch hand-written .cu files, not generated ones

vortex-datafusion/examples/vortex_table.rs

@@ -70,9 +70,7 @@ async fn main() -> anyhow::Result<()> {
             .ok_or_else(|| vortex_err!("Path is not valid UTF-8"))?,
     )?;
     let config = ListingTableConfig::new(table_url)
-        .with_listing_options(
-            ListingOptions::new(format).with_session_config_options(ctx.state().config()),
-        )
+        .with_listing_options(ListingOptions::new(format))
         .infer_schema(&ctx.state())
         .await?;
 

vortex-datafusion/src/convert/exprs.rs

@@ -15,12 +15,12 @@ use datafusion_expr::Operator as DFOperator;
 use datafusion_functions::core::getfield::GetFieldFunc;
 use datafusion_functions::string::octet_length::OctetLengthFunc;
 use datafusion_functions_nested::length::ArrayLength;
+use datafusion_physical_expr::DynamicFilterTracking;
 use datafusion_physical_expr::PhysicalExpr;
 use datafusion_physical_expr::ScalarFunctionExpr;
 use datafusion_physical_expr::projection::ProjectionExpr;
 use datafusion_physical_expr::projection::ProjectionExprs;
 use datafusion_physical_expr::utils::collect_columns;
-use datafusion_physical_expr_common::physical_expr::is_dynamic_physical_expr;
 use datafusion_physical_plan::expressions as df_expr;
 use itertools::Itertools;
 use vortex::VortexSessionDefault;
@@ -532,7 +532,7 @@ fn try_operator_from_df(value: &DFOperator) -> DFResult<Operator> {
 fn can_be_pushed_down_impl(expr: &Arc<dyn PhysicalExpr>, schema: &Schema) -> bool {
     // We currently do not support pushdown of dynamic expressions in DF.
     // See issue: https://github.com/vortex-data/vortex/issues/4034
-    if is_dynamic_physical_expr(expr) {
+    if DynamicFilterTracking::classify(expr).contains_dynamic_filter() {
         return false;
     }
 

vortex-datafusion/src/persistent/cache.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+use datafusion_common::HashMap;
 use datafusion_execution::cache::cache_manager::FileMetadata;
 use vortex::file::Footer;
 use vortex::file::VortexFile;
@@ -41,7 +42,7 @@ impl FileMetadata for CachedVortexMetadata {
     }
 
     #[expect(clippy::disallowed_types)]
-    fn extra_info(&self) -> std::collections::HashMap<String, String> {
+    fn extra_info(&self) -> HashMap<String, String> {
         Default::default()
     }
 }

vortex-datafusion/src/persistent/format.rs

@@ -533,7 +533,13 @@ impl FileFormat for VortexFormat {
                 })
                 .map(|f| f.vortex_expect("Failed to spawn infer_schema"))
             })
-            .buffer_unordered(state.config_options().execution.meta_fetch_concurrency)
+            .buffer_unordered(
+                state
+                    .config_options()
+                    .execution
+                    .meta_fetch_concurrency
+                    .get(),
+            )
             .try_collect::<Vec<_>>()
             .await
             .map_err(|e| DataFusionError::Execution(format!("Failed to infer schema: {e}")))?;
@@ -880,9 +886,7 @@ mod tests {
         format: &dyn FileFormat,
         calls: &ExpressionConvertorCalls,
     ) -> anyhow::Result<()> {
-        let source = format.file_source(TableSchema::from_file_schema(
-            expression_convertor_test_schema(),
-        ));
+        let source = format.file_source(TableSchema::from(expression_convertor_test_schema()));
         let result = source.try_pushdown_filters(
             vec![expression_convertor_test_filter()],
             &ConfigOptions::new(),
@@ -949,7 +953,7 @@ mod tests {
             ..Default::default()
         };
         let format = VortexFormat::new_with_options(VortexSession::default(), opts.clone());
-        let table_schema = TableSchema::from_file_schema(Arc::new(Schema::empty()));
+        let table_schema = TableSchema::from(Arc::new(Schema::empty()));
 
         let source = format.file_source(table_schema);
         let source = source