Skip to content

Commit 168fe49

Browse files
authored
Serialize parquet_options in datafusion-proto (#14465)
* Serialize `parquet_options` * Fix format
1 parent 68ee07f commit 168fe49

File tree

5 files changed

+35
-0
lines changed

5 files changed

+35
-0
lines changed

datafusion/proto/proto/datafusion.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1005,6 +1005,8 @@ message ParquetScanExecNode {
10051005
reserved 2;
10061006

10071007
PhysicalExprNode predicate = 3;
1008+
1009+
datafusion_common.TableParquetOptions parquet_options = 4;
10081010
}
10091011

10101012
message CsvScanExecNode {

datafusion/proto/src/generated/pbjson.rs

Lines changed: 18 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

datafusion/proto/src/generated/prost.rs

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

datafusion/proto/src/physical_plan/mod.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,11 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {
258258
})
259259
.transpose()?;
260260
let mut builder = ParquetExec::builder(base_config);
261+
262+
if let Some(options) = scan.parquet_options.as_ref() {
263+
builder = builder.with_table_parquet_options(options.try_into()?)
264+
}
265+
261266
if let Some(predicate) = predicate {
262267
builder = builder.with_predicate(predicate)
263268
}
@@ -1654,6 +1659,7 @@ impl AsExecutionPlan for protobuf::PhysicalPlanNode {
16541659
extension_codec,
16551660
)?),
16561661
predicate,
1662+
parquet_options: Some(exec.table_parquet_options().try_into()?),
16571663
},
16581664
)),
16591665
});

datafusion/proto/tests/cases/roundtrip_physical_plan.rs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -730,9 +730,14 @@ fn roundtrip_parquet_exec_with_pruning_predicate() -> Result<()> {
730730
Operator::Eq,
731731
lit("1"),
732732
));
733+
734+
let mut options = TableParquetOptions::new();
735+
options.global.pushdown_filters = true;
736+
733737
roundtrip_test(
734738
ParquetExec::builder(scan_config)
735739
.with_predicate(predicate)
740+
.with_table_parquet_options(options)
736741
.build_arc(),
737742
)
738743
}

0 commit comments

Comments
 (0)