Implement sort order aware redistribution

apache · Dec 13, 2023 · 0b558bc · 0b558bc
1 parent 0a9dfb9
commit 0b558bc
Show file tree

Hide file tree

Showing 5 changed files with 482 additions and 48 deletions.
diff --git a/datafusion/core/src/datasource/physical_plan/csv.rs b/datafusion/core/src/datasource/physical_plan/csv.rs
@@ -193,6 +193,7 @@ impl ExecutionPlan for CsvExec {
 
         let repartitioned_file_groups_option = FileGroupPartitioner::new()
             .with_target_partitions(target_partitions)
+            .with_preserve_order_within_groups(self.output_ordering().is_some())
             .with_repartition_file_min_size(repartition_file_min_size)
             .repartition_file_groups(&self.base_config.file_groups);