Datafusion 43-hstack

hstack · Dec 19, 2024 · 794f31d · 794f31d
1 parent 2c83fd8
commit 794f31d
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 2 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -84,6 +84,7 @@ datafusion-common-runtime = { git = 'https://github.com/hstack/arrow-datafusion.
 datafusion-execution = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
 datafusion-expr = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
 datafusion-expr-common = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
+datafusion-ffi = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
 datafusion-functions = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
 datafusion-functions-aggregate = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
 datafusion-functions-aggregate-common = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
@@ -92,7 +93,6 @@ datafusion-functions-window = { git = 'https://github.com/hstack/arrow-datafusio
 datafusion-optimizer = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
 datafusion-physical-expr = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
 datafusion-physical-expr-common = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
-datafusion-physical-expr-functions-aggregate = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
 datafusion-physical-optimizer = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
 datafusion-physical-plan = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }
 datafusion-proto = { git = 'https://github.com/hstack/arrow-datafusion.git', branch = 'main' }

diff --git a/crates/core/src/delta_datafusion/cdf/scan_utils.rs b/crates/core/src/delta_datafusion/cdf/scan_utils.rs
@@ -84,6 +84,7 @@ pub fn create_partition_values<F: FileAction>(
                 extensions: None,
                 range: None,
                 statistics: None,
+                metadata_size_hint: None,
             };
 
             file_groups.entry(new_part_values).or_default().push(part);

diff --git a/crates/core/src/delta_datafusion/mod.rs b/crates/core/src/delta_datafusion/mod.rs
@@ -1111,6 +1111,7 @@ fn partitioned_file_from_action(
         range: None,
         extensions: None,
         statistics: None,
+        metadata_size_hint: None,
     }
 }
 
@@ -2014,6 +2015,7 @@ mod tests {
             range: None,
             extensions: None,
             statistics: None,
+            metadata_size_hint: None,
         };
         assert_eq!(file.partition_values, ref_file.partition_values)
     }

diff --git a/crates/core/src/delta_datafusion/schema_adapter.rs b/crates/core/src/delta_datafusion/schema_adapter.rs
@@ -35,7 +35,7 @@ pub(crate) struct DeltaSchemaAdapter {
 
 impl SchemaAdapter for DeltaSchemaAdapter {
     fn map_column_index(&self, index: usize, file_schema: &Schema) -> Option<usize> {
-        let field = self.table_schema.field(index);
+        let field = self.projected_table_schema.field(index);
         Some(file_schema.fields.find(field.name())?.0)
     }